html2rss 0.27.1 → 0.28.0

data/lib/html2rss/feed_resolution/diag.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Wire-safe diagnostics for one entry-resolution attempt (Status / Marshal edge).
+    Diag = Data.define(:applied, :probe_count, :reason, :winner_score) do
+      ##
+      # @param result [Html2rss::FeedResolution::Result]
+      # @return [Diag]
+      def self.from_result(result)
+        new(
+          applied: result.applied,
+          probe_count: result.probe_count,
+          reason: result.reason,
+          winner_score: result.winner_score
+        )
+      end
+
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h
+        {
+          applied:,
+          probe_count:,
+          reason:,
+          **(winner_score.nil? ? {} : { winner_score: })
+        }
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution/options.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Typed +auto_source.entry_resolution+ options (one expansion from config Hash).
+    Options = Data.define(:enabled, :max_probes) do
+      ##
+      # @param auto_source [Hash, nil]
+      # @return [Options]
+      def self.from_auto_source(auto_source)
+        raw = auto_source.is_a?(Hash) ? auto_source[:entry_resolution] : nil
+        raw = {} unless raw.is_a?(Hash)
+        new(
+          enabled: raw.fetch(:enabled, FeedResolution::DEFAULT_CONFIG[:enabled]) != false,
+          max_probes: Integer(raw.fetch(:max_probes, FeedResolution::DEFAULT_CONFIG[:max_probes]))
+        )
+      end
+
+      ##
+      # @return [Boolean]
+      def enabled? = enabled
+
+      ##
+      # Probe slots plus one Orchestrator retry GET when enabled.
+      #
+      # @return [Integer]
+      def request_slots = enabled? ? max_probes + 1 : 0
+    end
+  end
+end

data/lib/html2rss/feed_resolution/policy.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Decides whether an entry URL tournament should run after a weak extract.
+    module Policy
+      # Minimum admitted articles before the entry URL is considered strong enough.
+      ARTICLE_FLOOR = 3
+
+      module_function
+
+      ##
+      # @param config [Html2rss::Config]
+      # @param articles [Array<Html2rss::Article>] typed extract results (count derived)
+      # @param surface_category [Html2rss::SurfaceCategory, Symbol, nil]
+      # @return [Boolean]
+      def resolve?(config:, articles:, surface_category:)
+        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+        return false unless eligible_config?(config)
+
+        category = SurfaceCategory.coerce(surface_category)
+        return false if category.blocked?
+
+        articles.size < ARTICLE_FLOOR || category.weak? || native_feed_majority?(articles)
+      end
+
+      def eligible_config?(config)
+        return false unless config.auto_source
+        return false if config.selectors
+
+        Options.from_auto_source(config.auto_source).enabled?
+      end
+      module_function :eligible_config?
+      private_class_method :eligible_config?
+
+      def native_feed_majority?(articles)
+        return false if articles.empty?
+
+        native = articles.count { |article| article.scraper == AutoSource::Scraper::NativeFeed }
+        native * 2 >= articles.size
+      end
+      module_function :native_feed_majority?
+      private_class_method :native_feed_majority?
+    end
+  end
+end

data/lib/html2rss/feed_resolution/probe.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Cheap follow-up probe that scores a candidate via {PageRecon.assess} or syndication parse.
+    class Probe
+      ##
+      # @param url [Html2rss::Url]
+      # @param score [Numeric]
+      # @param articles_count [Integer]
+      Scored = Data.define(:url, :score, :articles_count)
+
+      ##
+      # @param request_session [Html2rss::RequestSession]
+      # @param origin_url [String, Html2rss::Url]
+      def initialize(request_session:, origin_url:)
+        @request_session = request_session
+        @origin_url = Html2rss::Url.from_absolute(origin_url)
+      end
+
+      ##
+      # @param url [Html2rss::Url]
+      # @return [Scored, nil]
+      def call(url)
+        response = request_session.follow_up(
+          url:,
+          relation: :entry_resolution,
+          origin_url:
+        )
+        score_response(url, response)
+      rescue Html2rss::Error
+        nil
+      end
+
+      private
+
+      attr_reader :request_session, :origin_url
+
+      def score_response(_url, response) # rubocop:disable Metrics/MethodLength -- feed vs HTML branches
+        if response.feed_response?
+          count = Syndication::Parser.parse_response(response).size
+          return Scored.new(
+            url: response.url,
+            score: Scorer.score_feed(articles_count: count),
+            articles_count: count
+          )
+        end
+
+        assessment = PageRecon.assess(response:, url: response.url)
+        Scored.new(
+          url: response.url,
+          score: Scorer.score_assessment(assessment),
+          articles_count: assessment.articles_count
+        )
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution/scorer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Html2rss
+  module FeedResolution
+    ##
+    # Scores probe targets and picks winners from cheap {PageRecon::Assessment} facts
+    # or syndication item counts.
+    module Scorer
+      # Weight for admitted article counts on HTML probes.
+      ARTICLE_WEIGHT = 10
+      # Bonus when the surface looks like a listing rather than a hub/shell.
+      LISTING_SURFACE_BONUS = 5
+
+      module_function
+
+      ##
+      # @param articles_count [Integer]
+      # @return [Integer]
+      def score_feed(articles_count:)
+        articles_count.to_i * ARTICLE_WEIGHT
+      end
+
+      ##
+      # @param assessment [Html2rss::PageRecon::Assessment]
+      # @return [Integer]
+      def score_assessment(assessment)
+        score = assessment.articles_count * ARTICLE_WEIGHT
+        score += LISTING_SURFACE_BONUS if assessment.listing_bonus?
+        drops = assessment.admission_drops.values.sum
+        total = assessment.articles_count + drops
+        score -= ((drops.to_f / total) * ARTICLE_WEIGHT).round if total.positive?
+        score
+      end
+
+      ##
+      # @param scored [Array<Probe::Scored>]
+      # @param entry_articles_count [Integer]
+      # @return [Probe::Scored, nil]
+      def pick_winner(scored:, entry_articles_count:)
+        winner = scored.max_by(&:score)
+        return unless winner
+        return if winner.articles_count <= entry_articles_count.to_i
+        return if winner.score <= 0
+
+        winner
+      end
+    end
+  end
+end

data/lib/html2rss/feed_resolution.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Entry URL tournament for weak homepage/hub extracts.
+  #
+  # {include:file:lib/html2rss/feed_resolution/README.md}
+  module FeedResolution
+    # Default entry-resolution options merged into AutoSource::DEFAULT_CONFIG.
+    DEFAULT_CONFIG = {
+      enabled: true,
+      max_probes: 5
+    }.freeze
+
+    ##
+    # Tournament outcome for one entry-resolution attempt.
+    Result = Data.define(:entry_url, :scrape_url, :applied, :reason, :probe_count, :winner_score)
+
+    ##
+    # @param auto_source [Hash, nil]
+    # @return [Integer] entry-resolution budget slots (probes + retry GET when enabled)
+    def self.request_slots_for(auto_source)
+      return 0 unless auto_source
+
+      Options.from_auto_source(auto_source).request_slots
+    end
+
+    ##
+    # @param entry_url [String, Html2rss::Url]
+    # @param response [Html2rss::RequestService::Response]
+    # @param session [Html2rss::RequestSession]
+    # @param config [Html2rss::Config]
+    # @param articles [Array<Html2rss::Article>]
+    # @param surface_category [Symbol, nil]
+    # @return [Result]
+    # rubocop:disable Metrics/ParameterLists -- tournament kwargs stay co-located
+    def self.call(entry_url:, response:, session:, config:, articles:, surface_category: nil)
+      Runner.new(
+        entry_url:, response:, session:, config:, articles:, surface_category:
+      ).call
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    ##
+    # Outcome of {FeedResolution.try_apply!} for the auto-fallback chain.
+    ApplyOutcome = Data.define(:scrape_target, :status)
+
+    ##
+    # Runs the tournament and, when a winner applies, re-fetches and re-extracts articles.
+    #
+    # @param pipeline [Html2rss::FeedPipeline]
+    # @param config [Html2rss::Config]
+    # @param response [Html2rss::RequestService::Response]
+    # @param session [Html2rss::RequestSession]
+    # @param strategy [Symbol]
+    # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+    # @param articles [Array]
+    # @param scrape_target [Html2rss::ScrapeTarget]
+    # @param state [Html2rss::FeedPipeline::AutoFallback::AttemptState]
+    # @param budget [Html2rss::RequestService::Budget]
+    # @return [ApplyOutcome, nil] `:succeeded` sticky scrape target when retry extract yielded items;
+    #   `nil` when no winner or retry was empty (entry scrape target kept; Diag.applied may still be true)
+    # rubocop:disable Metrics/ParameterLists -- orchestration kwargs stay co-located
+    def self.try_apply!(pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+                        scrape_target:, state:, budget:)
+      Orchestrator.new(
+        pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+        scrape_target:, state:, budget:
+      ).call
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # Tournament + optional retry orchestration for {FeedPipeline::AutoFallback}.
+    class Orchestrator
+      # Minimum remaining request budget required before probing candidates.
+      MIN_BUDGET_REMAINING = 3
+
+      # Error classes that abort resolution instead of falling through (mirrors AutoFallback).
+      NON_FALLBACK_ERRORS = FeedPipeline::AutoFallback::NON_FALLBACK_ERRORS
+
+      ##
+      # @param pipeline [Html2rss::FeedPipeline]
+      # @param config [Html2rss::Config]
+      # @param response [Html2rss::RequestService::Response]
+      # @param session [Html2rss::RequestSession]
+      # @param strategy [Symbol]
+      # @param resources [Html2rss::FeedPipeline::RuntimePolicy::Resources]
+      # @param articles [Array]
+      # @param scrape_target [Html2rss::ScrapeTarget]
+      # @param state [Html2rss::FeedPipeline::AutoFallback::AttemptState]
+      # @param budget [Html2rss::RequestService::Budget]
+      # rubocop:disable Metrics/ParameterLists -- orchestration context stays co-located
+      def initialize(pipeline:, config:, response:, session:, strategy:, resources:, articles:,
+                     scrape_target:, state:, budget:)
+        @pipeline = pipeline
+        @config = config
+        @response = response
+        @session = session
+        @strategy = strategy
+        @resources = resources
+        @articles = articles
+        @scrape_target = scrape_target
+        @state = state
+        @budget = budget
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      ##
+      # @return [ApplyOutcome, nil]
+      # rubocop:disable Metrics/MethodLength -- eligibility + tournament + retry path
+      def call
+        return unless eligible?
+
+        state.mark_resolution_tried!
+        resolution = run_tournament
+        state.remember_entry_resolution(resolution)
+        return unless resolution.applied
+
+        effective = scrape_target.with_effective(resolution.scrape_url)
+        return ApplyOutcome.new(scrape_target: effective, status: :succeeded) if retry_with(
+          resolution, effective:
+        )
+
+        nil
+      rescue RequestService::RequestBudgetExceeded
+        Log.warn('FeedResolution: entry resolution skipped (request budget exhausted)')
+        nil
+      rescue *NON_FALLBACK_ERRORS
+        raise
+      rescue StandardError => error
+        Log.warn("FeedResolution: entry resolution retry failed (#{error.class})")
+        nil
+      end
+      # rubocop:enable Metrics/MethodLength
+
+      private
+
+      attr_reader :pipeline, :config, :response, :session, :strategy, :resources, :articles,
+                  :scrape_target, :state, :budget, :assessment
+
+      def eligible?
+        return false if state.resolution_tried?
+        return false if response.feed_response?
+        return false if budget.remaining_requests < MIN_BUDGET_REMAINING
+
+        @assessment = PageRecon.assess(response:, url: config.url)
+        Policy.resolve?(
+          config:, articles:, surface_category: assessment.surface_category
+        )
+      end
+
+      def run_tournament
+        FeedResolution.call(
+          entry_url: config.url,
+          response:,
+          session:,
+          config:,
+          articles:,
+          surface_category: assessment.surface_category
+        )
+      end
+
+      def retry_with(_resolution, effective:) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- retry extract path
+        retry_session = pipeline.request_session_for(
+          config, strategy:, resources:, scrape_url: effective.effective_url
+        )
+        retry_response = retry_session.fetch_initial_response
+        retry_articles, dedup_dropped, admission_drops = pipeline.deduplicated_articles(
+          config:, response: retry_response, request_session: retry_session
+        )
+        state.remember_response(retry_response)
+        state.record_items(
+          strategy:, items_count: retry_articles.size, transport_meta: retry_response.transport_meta
+        )
+        return unless retry_articles.size.positive?
+
+        state.succeed!(
+          response: retry_response, articles: retry_articles, dedup_dropped:,
+          selected_strategy: strategy, attempt_count: state.attempts.size, admission_drops:,
+          scrape_target: effective
+        )
+        true
+      end
+    end
+
+    # Internal tournament runner (keeps {FeedResolution} a Zeitwerk namespace module).
+    class Runner
+      ##
+      # @param entry_url [String, Html2rss::Url]
+      # @param response [Html2rss::RequestService::Response]
+      # @param session [Html2rss::RequestSession]
+      # @param config [Html2rss::Config]
+      # @param articles [Array<Html2rss::Article>]
+      # @param surface_category [Symbol, nil]
+      # rubocop:disable Metrics/ParameterLists -- tournament context stays co-located
+      def initialize(entry_url:, response:, session:, config:, articles:, surface_category: nil)
+        raise ArgumentError, 'articles must be an Array' unless articles.is_a?(Array)
+
+        @entry_url = Html2rss::Url.from_absolute(entry_url)
+        @response = response
+        @session = session
+        @config = config
+        @articles = articles
+        @surface_category = surface_category
+      end
+      # rubocop:enable Metrics/ParameterLists
+
+      ##
+      # @return [Result]
+      # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- policy → candidates → probe → select
+      def call
+        return skip(:policy_skip) unless Policy.resolve?(
+          config:, articles:, surface_category:
+        )
+
+        candidates = CandidateGenerator.call(
+          entry_url:,
+          response:,
+          max: max_probes
+        )
+        return skip(:no_candidates) if candidates.empty?
+
+        scored = probe_candidates(candidates)
+        winner = Scorer.pick_winner(scored:, entry_articles_count: articles.size)
+        return skip(:no_winner, probe_count: scored.size) unless winner
+
+        apply(winner, probe_count: scored.size)
+      end
+      # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
+
+      private
+
+      attr_reader :entry_url, :response, :session, :config, :articles, :surface_category
+
+      def max_probes
+        Options.from_auto_source(config.auto_source).max_probes
+      end
+
+      def probe_candidates(candidates)
+        probe = Probe.new(request_session: session, origin_url: entry_url)
+        candidates.filter_map { |url| probe.call(url) }
+      end
+
+      def apply(winner, probe_count:)
+        result(
+          applied: true,
+          reason: :winner,
+          scrape_url: winner.url.to_s,
+          probe_count:,
+          winner_score: winner.score
+        )
+      end
+
+      def skip(reason, probe_count: 0)
+        result(
+          applied: false,
+          reason:,
+          scrape_url: entry_url.to_s,
+          probe_count:,
+          winner_score: nil
+        )
+      end
+
+      def result(applied:, reason:, scrape_url:, probe_count:, winner_score:)
+        scrape_host = Html2rss::Url.from_absolute(scrape_url).host
+        Log.info(
+          "FeedResolution: entry_host=#{entry_url.host} scrape_host=#{scrape_host} " \
+          "applied=#{applied} probe_count=#{probe_count} winner_score=#{winner_score.inspect} reason=#{reason}"
+        )
+        Result.new(entry_url: entry_url.to_s, scrape_url:, applied:, reason:, probe_count:, winner_score:)
+      end
+    end
+  end
+end

data/lib/html2rss/mcp/inspect.rb

@@ -3,65 +3,28 @@
 module Html2rss
   module MCP
     ##
-    # Diagnostic inspect path (not Capture ownership). Fetches once for SST/scraper stats
-    # and recon facts (final URL, status, scheme downgrade, native feed hints).
-    module Inspect # rubocop:disable Metrics/ModuleLength -- diagnostic helpers stay co-located
+    # Diagnostic inspect path (not Capture ownership). Fetches once, then delegates
+    # shared recon to {Html2rss::PageRecon}; adds MCP-only scraper/XHR diagnostics.
+    module Inspect
       module_function
 
       ##
       # @param url [String]
       # @param strategy [String, Symbol]
       # @return [Hash]
-      def call(url:, strategy: :auto) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      def call(url:, strategy: :auto)
         resolved = FeedPipeline::StrategyPlan.concrete_for_diagnostic(strategy)
         response = fetch_response(url, resolved)
-        parsed = response.parsed_body
+        recon = PageRecon.call(response:, url:, strategy: resolved)
 
-        result = recon_fields(url, response, parsed).merge(
+        result = recon.to_h.merge(
           strategy: resolved,
-          content_type: response.content_type,
-          html_response: response.html_response?,
-          scraper_eligibility: scraper_info(parsed),
-          sst_stats: sst_stats_from(response)
+          scraper_eligibility: scraper_info(safe_parsed_body(response))
         )
-
-        if response.html_response?
-          sst = sst_document(response)
-          if sst
-            result[:sst] = {
-              node_count: sst.node_count,
-              degraded: sst.degraded,
-              segment_stats: segment_stats(sst, url)
-            }
-          end
-        end
-
-        blocked = Html2rss::RequestService::BlockedSurface.interstitial_signature_for(response.body)
-        result[:blocked_surface] = blocked[:key].to_s if blocked
         result[:xhr_capture] = xhr_capture_info(response) if resolved == :botasaurus
-        merge_admission_diagnostics!(result, response)
-
         result
       end
 
-      ##
-      # Surfaces Cleanup admission_drops without re-running full AutoSource discovery.
-      # Uses articles already extractable from a cheap AutoSource pass only when HTML.
-      #
-      # @param result [Hash]
-      # @param response [Html2rss::RequestService::Response]
-      # @return [void]
-      def merge_admission_diagnostics!(result, response)
-        return unless response.html_response?
-
-        source = AutoSource.new(response, AutoSource::DEFAULT_CONFIG.merge(limit: 10))
-        articles = source.articles
-        result[:articles_count] = articles.size
-        drops = source.admission_drops
-        result[:admission_drops] = drops if drops.any?
-      end
-      module_function :merge_admission_diagnostics!
-
       ##
       # @param response [Html2rss::RequestService::Response]
       # @return [Hash] redacted XHR capture diagnostics (no query strings)
@@ -147,95 +110,29 @@ module Html2rss
       def sst_stats_from(response)
         return nil unless response.html_response?
 
-        doc = sst_document(response)
-        return nil unless doc
+        recon = PageRecon.call(response:, url: response.url)
+        return nil unless recon.sst
 
-        { node_count: doc.node_count, degraded: doc.degraded }
-      rescue StandardError
-        nil
+        { node_count: recon.sst[:node_count], degraded: recon.sst[:degraded] }
       end
       module_function :sst_stats_from
 
-      ##
-      # @param response [Html2rss::RequestService::Response]
-      # @return [Html2rss::SST::Document, nil]
-      def sst_document(response)
-        Html2rss::SST::Normalizer.call(response.body)
-      rescue ArgumentError
-        nil
-      end
-      module_function :sst_document
-
-      ##
-      # @param sst [Html2rss::SST::Document]
-      # @param url [String]
-      # @return [Hash]
-      def segment_stats(sst, url)
-        segments = discover_segments(sst, url)
-        return { found: 0 } if segments.empty?
-
-        {
-          found: segments.size,
-          strategies: segments.map(&:strategy).uniq,
-          sample_paths: segments.first(5).map { |s| s.root_node.tag_path }
-        }
-      end
-      module_function :segment_stats
-
       ##
       # @param sst [Html2rss::SST::Document]
       # @param url [String]
       # @return [Array]
-      def discover_segments(sst, url)
-        link_resolver = Scoring::LinkResolver.new(url)
-        AutoSource::Segmenter.call(
-          sst,
-          base_url: url,
-          strategy: :list,
-          link_resolver:
-        )
-      rescue StandardError
-        []
-      end
+      def discover_segments(sst, url) = PageRecon.discover_segments(sst, url)
       module_function :discover_segments
 
-      ##
-      # @param requested_url [String]
-      # @param response [Html2rss::RequestService::Response]
-      # @param parsed [Object]
-      # @return [Hash]
-      def recon_fields(requested_url, response, parsed)
-        requested = Url.from_absolute(requested_url)
-        final = response.url
-
-        {
-          requested_url: requested.to_s,
-          final_url: final.to_s,
-          status: response.status,
-          scheme_downgrade: scheme_downgrade?(requested, final),
-          alternate_feeds: alternate_feeds_from(parsed)
-        }
-      end
-      module_function :recon_fields
-
-      ##
-      # @param requested [Html2rss::Url]
-      # @param final [Html2rss::Url]
-      # @return [Boolean] true when the fetch downgraded https to http
-      def scheme_downgrade?(requested, final)
-        requested.scheme == 'https' && final.scheme == 'http'
-      end
-      module_function :scheme_downgrade?
-
-      ##
-      # @param parsed [Object]
-      # @return [Array<Hash{Symbol => String}>]
-      def alternate_feeds_from(parsed)
-        return [] unless parsed.is_a?(Nokogiri::HTML::Document)
+      def safe_parsed_body(response)
+        return unless response.html_response?
 
-        Html::FeedLink.from_document(parsed).map { |link| { href: link.href, mime_type: link.mime_type } }
+        response.parsed_body
+      rescue RequestService::UnsupportedResponseContentType
+        nil
       end
-      module_function :alternate_feeds_from
+      module_function :safe_parsed_body
+      private_class_method :safe_parsed_body
     end
   end
 end

data/lib/html2rss/mcp/outcome.rb

@@ -24,7 +24,8 @@ module Html2rss
                        'scheme_downgrade, alternate_feeds).',
           validate_config: 'Call validate_config with payload.yaml or a config hash (XOR, not both).',
           apply_config: 'Call apply_config next. Confirm payload.item_count before shipping.',
-          scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus.',
+          scrape_url: 'Call scrape_url for articles now. strategy auto already runs Faraday then Botasaurus ' \
+                      'and promotes native RSS/Atom when present.',
           capture_config: 'Call capture_config for a reusable YAML draft, then follow next_step.',
           read_runtime: 'Read html2rss://runtime. Set BOTASAURUS_SCRAPER_URL on the MCP process ' \
                         'if botasaurus_configured is false.'
@@ -150,7 +151,8 @@ module Html2rss
         end
 
         def inspect_next_step(payload)
-          return NextStep.done if Array(payload[:alternate_feeds]).any?
+          # Runtime scrape_url now consumes native alternates (NativeFeed / direct feed parse).
+          return NextStep.scrape_url if Array(payload[:alternate_feeds]).any?
           return NextStep.capture_config if payload[:articles_count].to_i.positive?
 
           NextStep.scrape_url