html2rss 0.27.2 → 0.29.1

data/lib/html2rss/feed_resolution.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Entry URL tournament for weak homepage/hub extracts.
+  #
+  # {include:file:lib/html2rss/feed_resolution/README.md}
+  module FeedResolution
+    # Default entry-resolution options merged into AutoSource::DEFAULT_CONFIG.
+    DEFAULT_CONFIG = {
+      enabled: true,
+      max_probes: 5
+    }.freeze
+
+    ##
+    # Tournament outcome for one entry-resolution attempt.
+    Result = Data.define(:entry_url, :scrape_url, :applied, :reason, :probe_count, :winner_score)
+
+    ##
+    # @param auto_source [Hash, nil]
+    # @return [Integer] entry-resolution budget slots (probes + retry GET when enabled)
+    def self.request_slots_for(auto_source)
+      return 0 unless auto_source
+
+      Options.from_auto_source(auto_source).request_slots
+    end
+
+    ##
+    # @param entry_url [String, Html2rss::Url]
+    # @param response [Html2rss::RequestService::Response]
+    # @param session [Html2rss::RequestSession]
+    # @param config [Html2rss::Config]
+    # @param articles [Array<Html2rss::Article>]
+    # @param surface_category [Symbol, nil]
+    # @return [Result]
+    # rubocop:disable-next Metrics/ParameterLists -- tournament kwargs stay co-located
+    def self.call(entry_url:, response:, session:, config:, articles:, surface_category: nil)
+      Runner.new(
+        entry_url:, response:, session:, config:, articles:, surface_category:
+      ).call
+    end
+
+    ##
+    # Outcome of {FeedResolution.try_apply!} for the auto-fallback chain.
+    ApplyOutcome = Data.define(:scrape_target, :status)
+
+    ##
+    # Runs the tournament and, when a winner applies, re-fetches and re-extracts articles.
+    #
+    # @param pipeline [Html2rss::FeedPipeline]
+    # @param config [Html2rss::Config]
+    # @param response [Html2rss::RequestService::Response]
+    # @param session [Html2rss::RequestSession]
+    # @param strategy [Symbol]
+    # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+    # @param articles [Array]
+    # @param scrape_target [Html2rss::ScrapeTarget]
+    # @param state [Html2rss::FeedPipeline::AutoFallback::AttemptState]
+    # @param budget [Html2rss::RequestService::Budget]
+    # @return [ApplyOutcome, nil] `:succeeded` sticky scrape target when retry extract yielded items;
+    #   `nil` when no winner or retry was empty (entry scrape target kept; Diag.applied may still be true)
+    # rubocop:disable-next Metrics/ParameterLists -- orchestration kwargs stay co-located
+    def self.try_apply!(pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+                        scrape_target:, state:, budget:)
+      Orchestrator.new(
+        pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+        scrape_target:, state:, budget:
+      ).call
+    end
+
+    # Tournament + optional retry orchestration for {FeedPipeline::AutoFallback}.
+    class Orchestrator
+      # Minimum remaining request budget required before probing candidates.
+      MIN_BUDGET_REMAINING = 3
+
+      # Error classes that abort resolution instead of falling through (mirrors AutoFallback).
+      NON_FALLBACK_ERRORS = FeedPipeline::AutoFallback::NON_FALLBACK_ERRORS
+
+      ##
+      # @param pipeline [Html2rss::FeedPipeline]
+      # @param config [Html2rss::Config]
+      # @param response [Html2rss::RequestService::Response]
+      # @param session [Html2rss::RequestSession]
+      # @param strategy [Symbol]
+      # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+      # @param articles [Array]
+      # @param scrape_target [Html2rss::ScrapeTarget]
+      # @param state [Html2rss::FeedPipeline::AutoFallback::AttemptState]
+      # @param budget [Html2rss::RequestService::Budget]
+      # rubocop:disable-next Metrics/ParameterLists -- orchestration context stays co-located
+      def initialize(pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+                     scrape_target:, state:, budget:)
+        @pipeline = pipeline
+        @config = config
+        @response = response
+        @session = session
+        @strategy = strategy
+        @resources = resources
+        @articles = articles
+        @scrape_target = scrape_target
+        @state = state
+        @budget = budget
+      end
+
+      ##
+      # @return [ApplyOutcome, nil]
+      # rubocop:disable-next Metrics/MethodLength -- eligibility + tournament + retry path
+      def call
+        return unless eligible?
+
+        state.mark_resolution_tried!
+        resolution = run_tournament
+        state.remember_entry_resolution(resolution)
+        return unless resolution.applied
+
+        effective = scrape_target.with_effective(resolution.scrape_url)
+        return ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(
+          resolution, effective:
+        )
+
+        nil
+      rescue RequestService::RequestBudgetExceeded
+        Log.warn('FeedResolution: entry resolution skipped (request budget exhausted)')
+        nil
+      rescue *NON_FALLBACK_ERRORS
+        raise
+      rescue StandardError => error
+        Log.warn("FeedResolution: entry resolution retry failed (#{error.class})")
+        nil
+      end
+
+      private
+
+      attr_reader :pipeline, :config, :response, :session, :strategy, :resources, :articles,
+                  :scrape_target, :state, :budget, :assessment
+
+      def eligible?
+        return false if state.resolution_tried?
+        return false if response.feed_response?
+        return false if budget.remaining_requests < MIN_BUDGET_REMAINING
+
+        @assessment = PageRecon.assess(response:, url: config.url)
+        Policy.resolve?(
+          config:, articles:, surface_category: assessment.surface_category
+        )
+      end
+
+      def run_tournament
+        FeedResolution.call(
+          entry_url: config.url,
+          response:,
+          session:,
+          config:,
+          articles:,
+          surface_category: assessment.surface_category
+        )
+      end
+
+      def retry_with(_resolution, effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
+        retry_session = pipeline.request_session_for(
+          config, strategy:, resources:, scrape_url: effective.effective_url
+        )
+        retry_response = retry_session.fetch_initial_response
+        retry_articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
+          config:, response: retry_response, request_session: retry_session
+        )
+        state.remember_response(retry_response)
+        state.record_items(
+          strategy:, items_count: retry_articles.size, transport_meta: retry_response.transport_meta
+        )
+        return unless retry_articles.size.positive?
+
+        state.succeed!(
+          response: retry_response, articles: retry_articles, dedup_dropped:,
+          selected_strategy: strategy, attempt_count: state.attempts.size, admission_drops:,
+          scrape_target: effective
+        )
+        true
+      end
+    end
+
+    # Internal tournament runner (keeps {FeedResolution} a Zeitwerk namespace module).
+    class Runner
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param session [Html2rss::RequestSession]
+      # @param config [Html2rss::Config]
+      # @param articles [Array<Html2rss::Article>]
+      # @param surface_category [Symbol, nil]
+      # rubocop:disable-next Metrics/ParameterLists -- tournament context stays co-located
+      def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
+        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+
+        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @response = response
+        @session = session
+        @config = config
+        @articles = articles
+        @surface_category = surface_category
+      end
+
+      ##
+      # @return [Result]
+      # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength -- policy → candidates → probe → select
+      def call
+        return skip(:policy_skip) unless Policy.resolve?(
+          config:, articles:, surface_category:
+        )
+
+        candidates = CandidateGenerator.call(
+          entry_url:,
+          response:,
+          max: max_probes
+        )
+        return skip(:no_candidates) if candidates.empty?
+
+        scored = probe_candidates(candidates)
+        winner = Scorer.pick_winner(scored:, entry_articles_count: articles.size)
+        return skip(:no_winner, probe_count: scored.size) unless winner
+
+        apply(winner, probe_count: scored.size)
+      end
+
+      private
+
+      attr_reader :entry_url, :response, :session, :config, :articles, :surface_category
+
+      def max_probes
+        Options.from_auto_source(config.auto_source).max_probes
+      end
+
+      def probe_candidates(candidates)
+        probe = Probe.new(request_session: session, origin_url: entry_url)
+        candidates.filter_map { |url| probe.call(url) }
+      end
+
+      def apply(winner, probe_count:)
+        result(
+          applied: true,
+          reason: :winner,
+          scrape_url: winner.url.to_s,
+          probe_count:,
+          winner_score: winner.score
+        )
+      end
+
+      def skip(reason, probe_count: 0)
+        result(
+          applied: false,
+          reason:,
+          scrape_url: entry_url.to_s,
+          probe_count:,
+          winner_score: nil
+        )
+      end
+
+      def result(applied:, reason:, scrape_url:, probe_count:, winner_score:)
+        scrape_host = Html2rss::Url.from_absolute(scrape_url).host
+        Log.info(
+          "FeedResolution: entry_host=#{entry_url.host} scrape_host=#{scrape_host} " \
+          "applied=#{applied} probe_count=#{probe_count} winner_score=#{winner_score.inspect} reason=#{reason}"
+        )
+        Result.new(entry_url: entry_url.to_s, scrape_url:, applied:, reason:, probe_count:, winner_score:)
+      end
+    end
+  end
+end

data/lib/html2rss/html/article_extractor/heading_extractor.rb

@@ -19,26 +19,22 @@ module Html2rss
             tags = article_tag.css(HEADING_TAGS.join(','))
             if tags.any?
               select_best_heading(tags)
-            elsif fallback_anchorless && selected_anchor.nil?
-              fallback_heading(article_tag)
+            else
+              labeled = heading_from_aria_or_title(article_tag)
+              return labeled if labeled
+
+              fallback_heading(article_tag) if fallback_anchorless && selected_anchor.nil?
             end
           end
 
           private
 
           def select_best_heading(tags)
-            min_tag_name = tags.map(&:name).min
-            best_tag = nil
-            max_size = -1
-
-            tags.each do |tag|
-              next if tag.name != min_tag_name
-
-              size = Navigator::TextExtractor.call(tag)&.size.to_i
-              (best_tag = tag) && (max_size = size) if size > max_size
-            end
-
-            best_tag
+            tagged = tags.map { |tag| [Probe.tag(tag), tag] }
+            min_tag_name = tagged.map(&:first).min
+            tagged.select { |name, _tag| name == min_tag_name }
+                  .max_by { |_name, tag| Navigator::TextExtractor.call(tag)&.size.to_i }
+                  &.last
           end
 
           def fallback_heading(article_tag)
@@ -47,6 +43,20 @@ module Html2rss
             )
             fallback_tags.find { |t| !Navigator::TextExtractor.call(t).to_s.strip.empty? }
           end
+
+          def heading_from_aria_or_title(article_tag)
+            article_tag.css('[aria-label]').each do |node|
+              next if node['aria-label'].to_s.strip.empty?
+
+              return node
+            end
+            article_tag.css('[title]').each do |node|
+              next if node['title'].to_s.strip.empty?
+
+              return node
+            end
+            nil
+          end
         end
       end
     end

data/lib/html2rss/html/article_extractor.rb

@@ -5,7 +5,7 @@ module Html2rss
     ##
     # ArticleExtractor is responsible for extracting details (headline, url, images, etc.)
     # from an article_tag DOM node. DOM chrome helpers live on {Navigator}.
-    # rubocop:disable Metrics/ClassLength -- leftover re-extract stays with field extractors
+    # rubocop:disable-next Metrics/ClassLength -- leftover re-extract stays with field extractors
     class ArticleExtractor
       class << self
         ##
@@ -79,7 +79,7 @@ module Html2rss
         Url.from_relative("##{id}", base_url) if id
       end
 
-      # rubocop:disable Metrics/CyclomaticComplexity
+      # rubocop:disable-next Metrics/CyclomaticComplexity
       def extract_title
         source = heading || selected_anchor
         title_text = source ? Navigator.extract_visible_text(source) : fallback_anchorless_title
@@ -88,7 +88,6 @@ module Html2rss
         kicker = kicker_node ? Navigator.extract_visible_text(kicker_node).to_s.strip : nil
         kicker && !kicker.empty? && !title_text.include?(kicker) ? "#{kicker}: #{title_text}" : title_text
       end
-      # rubocop:enable Metrics/CyclomaticComplexity
 
       def fallback_anchorless_title
         return unless @fallback_anchorless && selected_anchor.nil?
@@ -130,11 +129,8 @@ module Html2rss
       end
 
       def heading_or_anchor_item?
-        heading_item? || article_tag.name.to_s == 'a'
-      end
-
-      def heading_item?
-        Navigator::HEADING_TAGS.include?(article_tag.name.to_s)
+        tag = Probe.tag(article_tag)
+        Navigator::HEADING_TAGS.include?(tag) || tag == 'a'
       end
 
       def heading_or_anchor_miss?(title, lines, published_at)
@@ -207,6 +203,5 @@ module Html2rss
 
       def extract_enclosures = EnclosureExtractor.call(article_tag, base_url)
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/html/feed_link.rb

@@ -25,8 +25,10 @@ module Html2rss
         end
 
         def feed_mime_type(type)
-          media_type = type.to_s.split(';', 2).first.strip.downcase
-          media_type if media_type in 'application/rss+xml' | 'application/atom+xml'
+          return Probe::APPLICATION_RSS_XML if Probe.mime_match?(type, Probe::APPLICATION_RSS_XML)
+          return Probe::APPLICATION_ATOM_XML if Probe.mime_match?(type, Probe::APPLICATION_ATOM_XML)
+
+          nil
         end
       end
     end

data/lib/html2rss/html/navigator.rb

@@ -11,9 +11,6 @@ module Html2rss
       # Element tags that indicate ignored DOM chrome when found in a container path.
       IGNORED_CONTAINER_TAGS = %w[nav footer header svg script style].to_set.freeze
 
-      # Layout roots and chrome tags excluded from class-clustering candidate nodes.
-      CLUSTER_EXCLUDED_TAGS = Set['html', 'body', 'nav', 'footer', 'header', 'svg', 'script', 'style'].freeze
-
       # Ancestor tags that usually indicate navigation/utility regions inside a content container.
       UTILITY_LANDMARK_TAGS = %w[nav aside footer menu].to_set.freeze
 
@@ -48,43 +45,11 @@ module Html2rss
         # @param article_tag [Nokogiri::XML::Node] article-like container to search within
         # @return [Nokogiri::XML::Node, nil] first eligible descendant anchor
         def main_anchor_for(article_tag)
-          return article_tag if article_tag.name == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
+          return article_tag if Probe.tag(article_tag) == 'a' && article_tag.matches?(MAIN_ANCHOR_SELECTOR)
 
           article_tag.at_css(MAIN_ANCHOR_SELECTOR)
         end
 
-        ##
-        # @param node [Nokogiri::XML::Node]
-        # @param cache [Hash, nil] identity cache used to store results (must use compare_by_identity)
-        # @return [Boolean] true when the node belongs to ignored DOM chrome
-        # rubocop:disable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-        def ignored_container_path?(node, cache = nil)
-          return cache[node] if cache&.key?(node)
-
-          curr = node
-          visited = []
-          is_ignored = false
-
-          while curr.respond_to?(:parent) && curr
-            if cache&.key?(curr)
-              is_ignored = cache[curr]
-              break
-            end
-
-            if IGNORED_CONTAINER_TAGS.include?(curr.name)
-              is_ignored = true
-              break
-            end
-
-            visited << curr
-            curr = curr.parent
-          end
-          visited.each { |n| cache[n] = is_ignored } if cache
-
-          is_ignored
-        end
-        # rubocop:enable Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
-
         ##
         # Returns the first parent that satisfies the condition.
         # If the condition is met, it returns the node itself.
@@ -93,7 +58,7 @@ module Html2rss
         # @param condition [Proc] The condition to be met.
         # @return [Nokogiri::XML::Node, nil] The first parent that satisfies the condition.
         def parent_until_condition(node, condition)
-          while node && !node.document? && node.name != 'html'
+          while node && !node.document? && Probe.tag(node) != 'html'
             return node if condition.call(node)
 
             node = node.parent
@@ -109,7 +74,7 @@ module Html2rss
           return false unless node
           return false if node.respond_to?(:document?) && node.document?
 
-          !CARD_WALK_STOP_TAGS.include?(node.name.to_s)
+          !CARD_WALK_STOP_TAGS.include?(Probe.tag(node))
         end
 
         ##
@@ -137,7 +102,7 @@ module Html2rss
         # @param tag_name [String] tag name to find in ancestors
         # @return [Nokogiri::XML::Node, nil] matching ancestor node
         def find_tag_in_ancestors(current_tag, tag_name)
-          return current_tag if current_tag.name == tag_name
+          return current_tag if Probe.tag(current_tag) == Probe.fold(tag_name)
 
           current_tag.ancestors(tag_name).first
         end

data/lib/html2rss/html/probe.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module Html
+    ##
+    # Case-insensitive DOM wire matching for MIME types, tag names, and link relations.
+    module Probe
+      # Canonical MIME type strings for script and alternate link matching.
+      APPLICATION_LD_JSON = 'application/ld+json'
+      # @see #mime_match?
+      APPLICATION_JSON = 'application/json'
+      # @see #mime_match?
+      APPLICATION_JSON_OEMBED = 'application/json+oembed'
+      # @see #mime_match?
+      APPLICATION_RSS_XML = 'application/rss+xml'
+      # @see #mime_match?
+      APPLICATION_ATOM_XML = 'application/atom+xml'
+
+      module_function
+
+      ##
+      # @param string [String, Symbol, #to_s] wire value
+      # @return [String] folded comparison key
+      def fold(string)
+        string.to_s.downcase
+      end
+
+      ##
+      # @param node [Nokogiri::XML::Node]
+      # @return [String] folded element name
+      def tag(node)
+        name = node.name
+        return fold(name) unless name.is_a?(String)
+        return name if name.ascii_only? && !name.match?(/[A-Z]/)
+
+        fold(name)
+      end
+
+      ##
+      # @param type [String, Symbol, #to_s, nil] MIME type attribute value
+      # @return [String] media type without parameters, folded
+      def mime_base(type)
+        raw = type.to_s
+        return fold(raw) unless raw.include?(';') || raw.match?(/\A\s|\s\z/)
+
+        fold(raw.split(';', 2).first.to_s.strip)
+      end
+
+      ##
+      # @param actual [String, Symbol, #to_s, nil] observed MIME type
+      # @param expected [Array<String>] canonical MIME types to match
+      # @return [Boolean]
+      def mime_match?(actual, *expected)
+        base = mime_base(actual)
+        expected.any? { |candidate| base == candidate || base == mime_base(candidate) }
+      end
+
+      ##
+      # @param doc [Nokogiri::XML::Node]
+      # @param mime_types [Array<String>] optional MIME filters (folded once per call)
+      # @return [Array<Nokogiri::XML::Element>]
+      def scripts(doc, *mime_types)
+        return doc.css('script') if mime_types.empty?
+
+        allowed = mime_types.to_set { |mime| mime_base(mime) }
+        doc.css('script').select { |script| allowed.include?(mime_base(script['type'])) }
+      end
+
+      ##
+      # @param doc [Nokogiri::XML::Node]
+      # @param rel [String] link rel token
+      # @param mime [String, nil] optional MIME filter
+      # @return [Array<Nokogiri::XML::Element>]
+      def alternate_links(doc, rel:, mime: nil)
+        nodes = doc.css(%(link[rel~="#{rel}"][href]))
+        return nodes if mime.nil?
+
+        nodes.select { |node| mime_match?(node['type'], mime) }
+      end
+    end
+  end
+end

data/lib/html2rss/html/sst_article_extractor.rb

@@ -7,7 +7,7 @@ module Html2rss
     ##
     # Builds an {Html2rss::Article} from an SST segment / ranked segment.
     # Port of Html::ArticleExtractor field logic onto SST::Node.
-    # rubocop:disable Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
+    # rubocop:disable-next Metrics/ClassLength -- SST field extractors colocated for parity with ArticleExtractor
     class SstArticleExtractor
       # CSS class tokens that mark kicker / eyebrow text (excluded from titles).
       KICKER_CLASS_PATTERN = /kicker|eyebrow|pre-title|pretitle|overline/i
@@ -390,6 +390,5 @@ module Html2rss
         end
       end
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/link_destination/noise_policy.rb

@@ -21,7 +21,7 @@ module Html2rss
       # @param heading_anchor [Boolean]
       # @param utility_landmark_ancestor [Boolean] pre-computed by Segmenter tree walk
       # @return [Boolean]
-      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
+      # rubocop:disable-next Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
       def noise_anchor?(text:, destination_facts:, anchor: nil, container: nil,
                         heading_anchor: false, utility_landmark_ancestor: false)
         return true unless destination_facts
@@ -35,7 +35,6 @@ module Html2rss
           icon_only_anchor?(anchor, text) ||
           utility_landmark_ancestor
       end
-      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/ParameterLists, Metrics/PerceivedComplexity, Lint/UnusedMethodArgument
 
       private