html2rss 0.24.0 → 0.25.0

data/lib/html2rss/link_destination/path_classifier.rb

@@ -0,0 +1,205 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    # Classifies normalized destination path segments for scoring.
+    class PathClassifier # rubocop:disable Metrics/ClassLength
+      attr_reader :segments
+
+      # Soft utility segments excluded from high-confidence junk (still utility_path).
+      SOFT_UTILITY = %w[
+        for-you join member members membership newsletter newsletters
+        plans plus premium pricing recommended
+      ].to_set.freeze
+
+      # Segment groups used to classify article, taxonomy, utility, and vanity routes.
+      SEGMENT_SETS = begin
+        content = %w[
+          article articles blog blogs changelog changelogs insight insights
+          launch launches news post posts release releases story stories update updates
+          artikel beitrag beitraege nachrichten neuigkeiten aktuelles
+          articulo articulos noticia noticias entrada entradas publicacion publicaciones
+          actualite actualites nouvelle nouvelles teaser teasers card cards
+        ].to_set.freeze
+        taxonomy = %w[
+          category categories tag tags topic topics
+          kategorie kategorien schlagwort schlagworte thema themen
+          categoria categorias etiqueta etiquetas tema temas
+          categorie etiquette etiquettes sujet sujets theme themes
+        ].to_set.freeze
+        vanity = %w[
+          join membership plus premium pricing plans subscribe signup
+          abonnieren abo suscribirse boletin s-abonner saboner
+        ].to_set.freeze
+        utility = (
+          taxonomy.to_a + %w[
+            about account archive archives author authors comment comments
+            contact feedback help login logout notification notifications
+            preference preferences profile register search settings share signup subscribe
+            feed feeds comment-feed comments-feed privacy terms cookie cookies user users
+            kategorie kategorien schlagwort schlagworte thema themen autor autoren archiv
+            ueber-uns ueber ueberuns profil kontakt impressum suche hilfe anmelden registrieren
+            konto registrierung anmeldung abonnieren abo datenschutz nutzungsbedingungen agb
+            categoria categorias etiqueta etiquetas tema temas autores archivos
+            sobre-nosotros sobre quienes-somos buscar busqueda ayuda entrar ingresar
+            registrarse registro cuenta suscribirse boletin privacidad condiciones
+            categorie etiquette etiquettes sujet sujets theme themes auteur auteurs
+            a-propos apropos recherche rechercher aide connexion s-inscrire
+            sinscrire inscription compte s-abonner saboner lettre-information confidentialite
+            mentions-legales cgu menu sidebar widget social modal popup banner promo ad ads
+            related recommendation recommendations pagination pager
+          ] + SOFT_UTILITY.to_a
+        ).to_set.freeze
+        {
+          content:,
+          utility:,
+          high_confidence_junk: (utility - SOFT_UTILITY).freeze,
+          taxonomy:,
+          vanity:,
+          deep_post_context: %w[press newsroom presse pressemitteilungen prensa].to_set.freeze
+        }.freeze
+      end
+      # Path segment that begins with a year-like publishing marker.
+      YEARISH_SEGMENT = /\A\d{4,}[\w-]*\z/
+      # Hyphenated slug shape common to article permalinks.
+      POST_SLUG_SEGMENT = /\A[a-z0-9]+(?:-[a-z0-9]+){2,}\z/i
+
+      # @param segments [Array<String>] normalized URL path segments
+      def initialize(segments)
+        @segments = segments
+      end
+
+      # @return [Boolean] true when the route has article-like path evidence
+      def content_path?
+        @content_path ||= SEGMENT_SETS[:content].intersect?(segments) ||
+                          yearish_content_context?
+      end
+
+      # @return [Boolean] true when the route includes utility/navigation evidence
+      def utility_path?
+        @utility_path ||= SEGMENT_SETS[:utility].intersect?(segments)
+      end
+
+      # @return [Boolean] true when the route points at conversion or account chrome
+      def vanity_path?
+        @vanity_path ||= SEGMENT_SETS[:vanity].intersect?(segments)
+      end
+
+      # @return [Boolean] true when the route points at taxonomy/listing chrome
+      def taxonomy_path?
+        @taxonomy_path ||= SEGMENT_SETS[:taxonomy].intersect?(segments)
+      end
+
+      # @return [Boolean] true when the route is too shallow to strongly indicate an article
+      def shallow?
+        segment_count = segments.size
+        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
+
+        segment_count <= 1 || (segment_count == 2 && junk_segments.include?(segments.last))
+      end
+
+      # @return [Boolean] true when the final path segment looks like a post slug
+      def strong_post_suffix?
+        @strong_post_suffix ||= segments.any? &&
+                                included_last_segment? &&
+                                trusted_post_context?(segments.size - 1)
+      end
+
+      # @return [Boolean] true when every path segment is utility chrome
+      def utility_only_route?
+        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
+
+        segments.all? { |segment| junk_segments.include?(segment) }
+      end
+
+      # @return [Boolean] true when the route is shallow and contains high-confidence noise
+      def shallow_high_confidence_route?
+        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
+        vanity_segments = SEGMENT_SETS.fetch(:vanity)
+
+        shallow? && segments.any? do |segment|
+          junk_segments.include?(segment) || vanity_segments.include?(segment)
+        end
+      end
+
+      # @return [Boolean] true when the leading segments are all utility chrome
+      def deep_utility_context_route?
+        all_junk?(segments.size - 1)
+      end
+
+      # @return [Boolean] true when the route is shallow and contains high-confidence noise
+      def junk_path?
+        return false if excluded_content_route?
+
+        taxonomy_path? ||
+          utility_only_route? ||
+          deep_utility_context_route? ||
+          shallow_high_confidence_route?
+      end
+
+      # @return [Boolean] true when the route points at conversion or account chrome
+      def utility_destination?
+        return false if excluded_content_route?
+
+        vanity_path? || utility_route?
+      end
+
+      private
+
+      def yearish_content_context?
+        segments.any? { |segment| segment.match?(YEARISH_SEGMENT) } &&
+          (strong_post_suffix? || trusted_post_context?(segments.size - 1))
+      end
+
+      def excluded_content_route?
+        segments.empty? || content_path? || strong_post_suffix?
+      end
+
+      def utility_route?
+        taxonomy_path? ||
+          utility_only_route? ||
+          deep_utility_context_route? ||
+          shallow_utility_route?
+      end
+
+      def shallow_utility_route?
+        shallow? && utility_path?
+      end
+
+      def all_junk?(limit)
+        return false if limit <= 0
+
+        junk_segments = SEGMENT_SETS.fetch(:high_confidence_junk)
+        (0...limit).all? { |i| junk_segments.include?(segments[i]) }
+      end
+
+      def trusted_post_context?(limit)
+        return false if limit <= 0
+
+        content_segments = SEGMENT_SETS.fetch(:content)
+        context_segments = SEGMENT_SETS.fetch(:deep_post_context)
+
+        (0...limit).any? do |i|
+          segment = segments[i]
+          content_segments.include?(segment) ||
+            segment.match?(PathClassifier::YEARISH_SEGMENT) ||
+            context_segments.include?(segment)
+        end
+      end
+
+      def included_last_segment?
+        !excluded_last_segment? && slug_last_segment?
+      end
+
+      def excluded_last_segment?
+        last = segments.last
+        [SEGMENT_SETS[:high_confidence_junk], SEGMENT_SETS[:vanity]].any? { |set| set.include?(last) }
+      end
+
+      def slug_last_segment?
+        last = segments.last
+        last.match?(YEARISH_SEGMENT) || last.match?(POST_SLUG_SEGMENT)
+      end
+    end
+  end
+end

data/lib/html2rss/link_destination/text_classifier.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module LinkDestination
+    # Classifies visible anchor text for utility and recommendation chrome.
+    class TextClassifier
+      # Prefix labels that usually identify navigation or subscription links.
+      UTILITY_PREFIX_PATTERN = /
+        \A\s*(
+          # English
+          view\s+all|see\s+all|all\s+news|subscribe|newsletter|comment\s+feed|comments\s+feed|join|premium|plus|
+          # German
+          alle\s+anzeigen|alle\s+news|abonnieren|newsletter|kommentar\s+feed|mitmachen|
+          # Spanish
+          ver\s+todos|ver\s+todo|todas\s+las\s+noticias|suscribirse|bolet(i|í)n|comentarios\s+feed|unirse|
+          # French
+          voir\s+tout|voir\s+tous|toutes\s+les\s+nouvelles|s['’]abonner|flux\s+de\s+commentaires|rejoindre
+        )\b
+      /ix
+      # Short labels that usually identify non-article navigation links.
+      UTILITY_PATTERN = /
+        \A\s*(
+          # English
+          about|contact|comments?|join|log\s+in|login|member(ship)?|
+          plus|premium|pricing|recommended(\s+for\s+you)?|
+          see\s+all|share|sign\s+up|signup|subscribe|view\s+all|
+          # German
+          (ue|ü)ber(\s+uns)?|kontakt|kommentare?|mitmachen|anmelden|login|
+          mitglied(schaft)?|empfohlen(\s+f(ue|ü)r\s+dich)?|alle\s+anzeigen|
+          teilen|registrieren|abonnieren|newsletter|
+          # Spanish
+          sobre(\s+nosotros)?|contacto|comentarios?|unirse|iniciar\s+sesion|
+          login|miembro|membres(i|í)a|recomendado(\s+para\s+ti)?|ver\s+todo|
+          compartir|registrarse|suscribirse|bolet(i|í)n|
+          # French
+          (a|à)\s+propos|(a|à)propos|contact|commentaires?|rejoindre|
+          se\s+connecter|login|membre|abonnement|recommand(e|é)(\s+pour\s+vous)?|
+          voir\s+tout|partager|s['’]inscrire|s['’]abonner|newsletter
+        )\b
+      /ix
+      # Labels for recommendation chrome rather than source articles.
+      RECOMMENDED_PATTERN = /
+        \A\s*(
+          recommended(\s+for\s+you)?|
+          empfohlen(\s+f(ue|ü)r\s+dich)?|
+          recomendado(\s+para\s+ti)?|
+          recommand(e|é)(\s+pour\s+vous)?
+        )\b
+      /ix
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text matches a utility label
+      def utility?(text) = text.to_s.match?(UTILITY_PATTERN)
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text begins with a utility label
+      def utility_prefix?(text) = text.to_s.match?(UTILITY_PREFIX_PATTERN)
+
+      # @param text [String, #to_s] visible anchor text
+      # @return [Boolean] true when text identifies recommendation chrome
+      def recommended?(text) = text.to_s.match?(RECOMMENDED_PATTERN)
+    end
+  end
+end

data/lib/html2rss/link_destination.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Destination URL / anchor-text classification shared by Scoring.
+  module LinkDestination
+  end
+end

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -66,14 +66,55 @@ module Html2rss
         faraday_request(response_guard, deadline:, streaming_buffer: false, consume_budget: false)
       end
 
+      ##
+      # Validates the remote socket peer IP during Net::HTTP start.
+      module PeerIpValidator
+        module_function
+
+        # @param http [Net::HTTP] connection to configure
+        # @param policy [Policy] request policy
+        # @return [void]
+        def install!(http, policy:)
+          orig_start = http.method(:start)
+          http.define_singleton_method(:start) do |&block|
+            orig_start.call do |opened_http|
+              PeerIpValidator.validate!(opened_http, policy:)
+              block.call(opened_http)
+            end
+          end
+        end
+
+        # @param opened_http [Net::HTTP] active connection
+        # @param policy [Policy] request policy
+        # @return [void]
+        def validate!(opened_http, policy:)
+          scheme = opened_http.use_ssl? ? 'https' : 'http'
+          url = Html2rss::Url.from_absolute("#{scheme}://#{opened_http.address}:#{opened_http.port}")
+          policy.validate_remote_ip!(ip: peer_ip_for(opened_http), url:)
+        end
+
+        # @param opened_http [Net::HTTP]
+        # @return [String, nil]
+        def peer_ip_for(opened_http)
+          sock = opened_http.instance_variable_get(:@socket)
+          io = sock.respond_to?(:io) ? sock.io : sock
+          peeraddr = io.respond_to?(:peeraddr) ? io.peeraddr : nil
+          peeraddr&.[](3) || peeraddr&.[](2)
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
       def client
         @client ||= Faraday.new(url: ctx.url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
           faraday.request :gzip
           faraday.use StreamingBodyMiddleware
-          faraday.adapter Faraday.default_adapter
+          faraday.adapter Faraday.default_adapter do |http|
+            PeerIpValidator.install!(http, policy: ctx.policy)
+          end
         end
       end
+      # rubocop:enable Metrics/AbcSize
 
       def apply_timeouts(request, deadline:)
         remaining_timeout = remaining_timeout_seconds(deadline)

data/lib/html2rss/request_service/network_guard.rb

@@ -48,7 +48,10 @@ module Html2rss
       def enforce_public_network!(url)
         host = url.host
         return if allow_private_networks?
-        return unless blocked_host?(host) || resolved_ip_addresses(host).any? { |address| blocked_ip?(address) }
+        raise PrivateNetworkDenied, "Private network target denied for #{url}" if blocked_host?(host)
+
+        addresses = resolved_ip_addresses(host)
+        return unless addresses.empty? || addresses.any? { |address| blocked_ip?(address) }
 
         raise PrivateNetworkDenied, "Private network target denied for #{url}"
       end
@@ -62,9 +65,8 @@ module Html2rss
       # @raise [PrivateNetworkDenied] if the response came from a blocked address
       def validate_remote_ip!(ip:, url:)
         return if allow_private_networks?
-        return if ip.nil? || ip.empty?
 
-        parsed_ip = parse_ip(ip)
+        parsed_ip = ip && !ip.to_s.empty? ? parse_ip(ip) : nil
         raise PrivateNetworkDenied, "Remote IP could not be validated for #{url}" unless parsed_ip
         return unless blocked_ip?(parsed_ip)
 

data/lib/html2rss/scoring/anchor_score.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Primary-link ranking weights (sole home; FeatureId-aligned).
+    module AnchorScore
+      # Signal → integer weight map for primary-link ranking.
+      WEIGHTS = {
+        heading_anchor: 100,
+        heading_text_match: 20,
+        meaningful_text: 10,
+        content_like_destination: 10
+      }.freeze
+
+      module_function
+
+      ##
+      # @param heading_anchor [Boolean]
+      # @param heading_text_match [Boolean]
+      # @param meaningful_text [Boolean]
+      # @param content_like_destination [Boolean]
+      # @return [Integer]
+      def score(heading_anchor:, heading_text_match:, meaningful_text:, content_like_destination:)
+        total = 0
+        total += WEIGHTS[:heading_anchor] if heading_anchor
+        total += WEIGHTS[:heading_text_match] if heading_text_match
+        total += WEIGHTS[:meaningful_text] if meaningful_text
+        total += WEIGHTS[:content_like_destination] if content_like_destination
+        total
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/cluster_scorer.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Scores DomClustering-style node groups (heading/time/date/word density).
+    # Sole home for cluster-group ranking weights.
+    class ClusterScorer
+      # Minimum average visible words before a group is eligible.
+      MIN_AVG_WORDS = 5
+      # Bonus when any node has a heading or bold title class.
+      HEADING_BONUS = 20
+      # Bonus when any node has a time/datetime marker.
+      TIME_BONUS = 20
+      # Bonus when any node has date-like visible text.
+      DATE_BONUS = 40
+
+      def initialize
+        @text_words = {}.compare_by_identity
+        @has_date = {}.compare_by_identity
+      end
+
+      ##
+      # @param groups [Hash{String => Array<SST::Node>}]
+      # @return [Array<SST::Node>]
+      def select_best_group(groups)
+        best_nodes = []
+        best_score = -1
+
+        groups.each_value do |nodes|
+          score = score_group(nodes)
+          next if score.negative?
+
+          (best_nodes = nodes) && (best_score = score) if score > best_score
+        end
+        best_nodes
+      end
+
+      ##
+      # @param nodes [Array<SST::Node>]
+      # @return [Float]
+      def avg_words(nodes)
+        nodes.sum { |n| text_words(n) } / nodes.size.to_f
+      end
+
+      private
+
+      def score_group(nodes)
+        avg_w = avg_words(nodes)
+        return -1 if avg_w < MIN_AVG_WORDS
+
+        score = nodes.size + (avg_w / 5.0)
+        score += HEADING_BONUS if nodes_heading?(nodes)
+        score += TIME_BONUS if nodes_time?(nodes)
+        score += DATE_BONUS if nodes_date?(nodes)
+        score
+      end
+
+      def nodes_heading?(nodes)
+        nodes.any? do |n|
+          n.find(&:heading?) ||
+            n.attrs.class_names.any? { |c| c.match?(/font-bold|font-semibold/) }
+        end
+      end
+
+      def nodes_time?(nodes)
+        nodes.any? { |n| n.find { |d| d.name == :time || d.attrs.datetime } }
+      end
+
+      def nodes_date?(nodes)
+        nodes.any? { |n| date?(n) }
+      end
+
+      def text_words(node)
+        @text_words[node] ||= node.word_count
+      end
+
+      def date?(node)
+        @has_date[node] ||= begin
+          text = node.visible_text.to_s
+          text.match?(%r{\b\d{4}[-/]\d{2}[-/]\d{2}\b}) ||
+            text.match?(/\b(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\b/i)
+        end
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/container_assessor.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Observes an SST container and builds typed {Observation} score inputs.
+    class ContainerAssessor
+      # Microdata itemprop values treated as publish/update markers.
+      PUBLISH_ITEMPROPS = %w[datePublished dateModified].freeze
+
+      # Matches content-like tokens in class/id strings.
+      CONTENT_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:content)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # Matches utility/junk tokens in class/id strings.
+      JUNK_TOKEN_REGEXP = begin
+        words = LinkDestination::PathClassifier::SEGMENT_SETS.fetch(:utility)
+        /(?:^|\s|[-_])(#{Regexp.union(words.to_a).source})(?:\s|[-_]|$)/i
+      end.freeze
+
+      # @param text_classifier [LinkDestination::TextClassifier]
+      def initialize(text_classifier: LinkDestination::TextClassifier.new)
+        @text_classifier = text_classifier
+      end
+
+      ##
+      # @param container [SST::Node]
+      # @param selected_anchor [SST::Node, nil]
+      # @param destination_facts [LinkDestination::DestinationFacts, nil]
+      # @return [Observation]
+      def call(container, selected_anchor, destination_facts:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
+        title = entry_title(container, selected_anchor)
+        tokens = "#{container.attrs.class_attr} #{container.attrs.id}"
+
+        Observation.new(
+          title_word_count: word_count(title),
+          path_length: destination_facts&.url&.path.to_s.length,
+          content_path: destination_facts&.content_path,
+          publish_marker: publish_marker?(container),
+          descriptive_context: descriptive_context?(container.visible_text, title),
+          article_container: container.name == :article,
+          content_tokens: tokens.match?(CONTENT_TOKEN_REGEXP),
+          junk_tokens: tokens.match?(JUNK_TOKEN_REGEXP),
+          utility_prefix_title: @text_classifier.utility_prefix?(title),
+          recommended_title: @text_classifier.recommended?(title),
+          utility_path: destination_facts&.utility_path,
+          strong_post_suffix: destination_facts&.strong_post_suffix,
+          shallow: destination_facts&.shallow,
+          high_confidence_junk_path: destination_facts&.high_confidence_junk_path,
+          high_confidence_utility_destination: destination_facts&.high_confidence_utility_destination,
+          selected_anchor_present: !selected_anchor.nil?
+        )
+      end
+
+      private
+
+      def publish_marker?(container)
+        !!container.find do |n|
+          n.name == :time ||
+            n.attrs.datetime ||
+            PUBLISH_ITEMPROPS.include?(n.attrs.itemprop.to_s)
+        end
+      end
+
+      def descriptive_context?(container_text, title)
+        snippet = container_text.to_s.sub(/\A#{Regexp.escape(title.to_s)}/i, '')
+        snippet.length > 30 && word_count(snippet) >= 8
+      end
+
+      def entry_title(container, selected_anchor)
+        heading = container.find(&:heading?)
+        source = heading || selected_anchor
+        source ? source.visible_text.to_s.strip : ''
+      end
+
+      def word_count(text)
+        text.to_s.scan(/\p{Alnum}+/).size
+      end
+    end
+  end
+end

data/lib/html2rss/scoring/engine.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Scoring
+    ##
+    # Ranks AutoSource::Segment instances with an explicit feature registry (no send).
+    class Engine
+      # Soft cap after score-floor filtering (lazy extraction guard; breadth > blind top-50).
+      TOP_K = 99
+      # Minimum composite score retained by {#rank_top} (precision floor).
+      SCORE_FLOOR = 0.0
+
+      # Quality feature triples: [FeatureId, predicate, weight].
+      QUALITY_RULES = [
+        [:title_word_count_ge3, ->(o) { o.title_word_count >= 3 }, 40],
+        [:title_word_count_ge7, ->(o) { o.title_word_count >= 7 }, 15],
+        [:path_length_gt6, ->(o) { o.path_length > 6 }, 20],
+        [:content_path, lambda(&:content_path), 15],
+        [:publish_marker, lambda(&:publish_marker), 15],
+        [:descriptive_context, lambda(&:descriptive_context), 10],
+        [:article_container, lambda(&:article_container), 10],
+        [:content_tokens, lambda(&:content_tokens), 10]
+      ].freeze
+
+      # Junk feature triples: [FeatureId, predicate, weight].
+      JUNK_RULES = [
+        [:non_content_utility_path, ->(o) { o.utility_path && !o.content_path && !o.strong_post_suffix }, 25],
+        [:utility_prefix_title_short, ->(o) { o.utility_prefix_title && o.title_word_count <= 6 }, 15],
+        [:shallow, lambda(&:shallow), 10],
+        [:weak_container, ->(o) { !o.publish_marker && !o.descriptive_context }, 10],
+        [:recommended_title_non_content, ->(o) { o.recommended_title && !o.content_path }, 10],
+        [:high_confidence_junk_path, lambda(&:high_confidence_junk_path), 5],
+        [:junk_tokens, lambda(&:junk_tokens), 15]
+      ].freeze
+
+      (QUALITY_RULES + JUNK_RULES).each { |feature_id, _, _| FeatureId.assert!(feature_id) }
+
+      # @param link_resolver [LinkResolver]
+      def initialize(link_resolver:)
+        @link_resolver = link_resolver
+        @assessor = ContainerAssessor.new(text_classifier: link_resolver.text_classifier)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @return [Array<RankedSegment>] sorted by composite desc, position asc; hard-junk dropped
+      def rank(segments)
+        segments.filter_map { |segment| rank_one(segment) }
+                .sort_by { |ranked| [-ranked.final_score, ranked.position] }
+      end
+
+      ##
+      # Filters hard-junk but preserves discovery order (list/cluster scrapers).
+      #
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>]
+      def select_eligible(segments, limit: TOP_K)
+        segments.filter_map { |segment| rank_one(segment, hard_junk_mode: :lenient) }
+                .sort_by(&:position)
+                .first(limit)
+      end
+
+      ##
+      # @param segments [Array<Html2rss::AutoSource::Segment>]
+      # @param limit [Integer]
+      # @return [Array<RankedSegment>] score-floor filtered, then capped
+      def rank_top(segments, limit: TOP_K)
+        rank(segments).select { |ranked| ranked.final_score >= SCORE_FLOOR }.first(limit)
+      end
+
+      private
+
+      def rank_one(segment, hard_junk_mode: :strict) # rubocop:disable Metrics/MethodLength
+        facts = segment.primary_link ? @link_resolver.destination_facts(segment.primary_link) : nil
+        obs = @assessor.call(segment.root_node, segment.primary_link, destination_facts: facts)
+        return if obs.hard_junk?(mode: hard_junk_mode)
+
+        quality, quality_parts = apply_rules(QUALITY_RULES, obs)
+        junk, junk_parts = apply_rules(JUNK_RULES, obs)
+        RankedSegment.build(
+          segment:,
+          score: Score.build(
+            composite: quality - junk,
+            quality:,
+            junk:,
+            breakdown: quality_parts.merge(junk_parts)
+          )
+        )
+      end
+
+      def apply_rules(rules, obs)
+        parts = {}
+        rules.each do |feature_id, predicate, weight|
+          parts[feature_id] = weight if predicate.call(obs)
+        end
+        [parts.values.sum, parts]
+      end
+    end
+  end
+end