html2rss 0.27.2 → 0.29.1

data/lib/html2rss/page_recon.rb

@@ -0,0 +1,297 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared page recon for MCP Inspect, curation Recon, and FeedResolution probes.
+  #
+  # Owns surface class, native feed hints, segment stats, and a cheap AutoSource
+  # article count. Diagnostic fetch for Inspect/Recon lives on {.probe} — not twin
+  # fetch helpers in those callers.
+  class PageRecon # rubocop:disable Metrics/ClassLength -- recon bag stays co-located
+    ##
+    # Cheap surface + admission facts shared by AutoFallback gates and FeedResolution probes.
+    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response) do
+      ##
+      # @return [Html2rss::SurfaceCategory]
+      def category = SurfaceCategory.coerce(surface_category)
+
+      ##
+      # @return [Boolean]
+      def weak? = category.weak?
+
+      ##
+      # @return [Boolean]
+      def blocked? = category.blocked?
+
+      ##
+      # @return [Boolean]
+      def listing_bonus? = category.listing_bonus?
+    end
+    ##
+    # Recon facts used by Inspect and FeedResolution.
+    Result = Data.define(
+      :requested_url,
+      :final_url,
+      :status,
+      :scheme_downgrade,
+      :alternate_feeds,
+      :surface_category,
+      :articles_count,
+      :admission_drops,
+      :segment_stats,
+      :html_response,
+      :content_type,
+      :blocked_surface,
+      :sst
+    ) do
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- omit-empty optional keys
+        {
+          requested_url:,
+          final_url:,
+          status:,
+          scheme_downgrade:,
+          alternate_feeds:,
+          surface_category:,
+          articles_count:,
+          html_response:,
+          content_type:,
+          **(admission_drops.any? ? { admission_drops: } : {}),
+          **(segment_stats ? { segment_stats: } : {}),
+          **(blocked_surface ? { blocked_surface: } : {}),
+          **(sst ? { sst: } : {})
+        }
+      end
+    end
+
+    ##
+    # Diagnostic fetch + assess bundle. One home for session build + initial GET + {call}.
+    # Consumed by curation {Html2rss::Recon} and {Html2rss::PageRecon::Diagnostics}.
+    Probe = Data.define(:session, :response, :result, :strategy)
+
+    ##
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url] requested entry URL
+    # @param strategy [Symbol, nil] unused (reserved for callers that already chose a strategy)
+    # @return [Result]
+    def self.call(response:, url:, strategy: nil) # rubocop:disable Lint/UnusedMethodArgument
+      new(response:, url:).call
+    end
+
+    ##
+    # Builds a request session, fetches the URL once, and runs full page recon.
+    #
+    # @param url [String, Html2rss::Url]
+    # @param strategy [Symbol] request strategy (:auto resolves to a concrete diagnostic strategy)
+    # @option options [Integer, nil] :max_redirects
+    # @option options [Integer, nil] :max_requests
+    # @return [Probe]
+    def self.probe(url, strategy: :auto, **)
+      url_obj = Url.from_absolute(url)
+      resolved = FeedPipeline::StrategyPlan.concrete_for_diagnostic(strategy)
+      session = build_probe_session(url_obj, resolved, **)
+      response = session.fetch_initial_response
+      Probe.new(
+        session:,
+        response:,
+        result: call(response:, url: url_obj, strategy: resolved),
+        strategy: resolved
+      )
+    end
+
+    ##
+    # @param url_obj [Html2rss::Url]
+    # @param strategy [Symbol]
+    # @option options [Integer, nil] :max_redirects
+    # @option options [Integer, nil] :max_requests
+    # @return [Html2rss::RequestSession]
+    def self.build_probe_session(url_obj, strategy, **options) # rubocop:disable Metrics/MethodLength
+      raw_config = Config.auto_source_config(
+        url: url_obj.to_s,
+        request_controls: Config::RequestControls.from_shortcut(
+          strategy:,
+          max_redirects: options[:max_redirects],
+          max_requests: options[:max_requests]
+        )
+      )
+      raw_config[:strategy] = strategy
+      config = Config.from_hash(raw_config)
+      resources = FeedPipeline::RuntimePolicy.resources_for(config)
+      RequestSession.build(
+        config:,
+        strategy: config.strategy,
+        budget: resources.budget,
+        policy: resources.policy
+      )
+    end
+    private_class_method :build_probe_session
+
+    ##
+    # Cheap page assessment for policy gates and probe scoring (fixed AutoSource limit).
+    #
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    # @return [Assessment]
+    def self.assess(response:, url:)
+      new(response:, url:).assess
+    end
+
+    ##
+    # Surface class only — no AutoSource extract (for empty-extract error labels).
+    #
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    # @return [Symbol]
+    def self.surface_category_for(response:, url:)
+      new(response:, url:).surface_category_for
+    end
+
+    ##
+    # @param sst [Html2rss::SST::Document]
+    # @param url [String, Html2rss::Url]
+    # @return [Array]
+    def self.discover_segments(sst, url)
+      link_resolver = Scoring::LinkResolver.new(url)
+      AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
+    rescue StandardError
+      []
+    end
+
+    ##
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    def initialize(response:, url:)
+      @response = response
+      @url = url
+    end
+
+    ##
+    # @return [Assessment]
+    def assess
+      return feed_assessment if response.feed_response?
+
+      parsed = html_parsed_body
+      articles_count, admission_drops = cheap_articles
+      Assessment.new(
+        surface_category: surface_category(parsed),
+        articles_count:,
+        admission_drops:,
+        html_response: true
+      )
+    end
+
+    ##
+    # @return [Symbol]
+    def surface_category_for
+      return :unsupported_surface if response.feed_response?
+
+      surface_category(html_parsed_body)
+    end
+
+    ##
+    # @return [Result]
+    def call # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- assemble recon Result
+      requested = Url.from_absolute(url)
+      final = response.url
+      parsed = html_parsed_body
+      assessment = assess
+      sst_payload, segment_stats = sst_payload_and_segments(requested)
+
+      Result.new(
+        requested_url: requested.to_s,
+        final_url: final.to_s,
+        status: response.status,
+        scheme_downgrade: scheme_downgrade?(requested, final),
+        alternate_feeds: alternate_feeds_from(parsed),
+        surface_category: assessment.surface_category,
+        articles_count: assessment.articles_count,
+        admission_drops: assessment.admission_drops,
+        segment_stats:,
+        html_response: response.html_response?,
+        content_type: response.content_type,
+        blocked_surface: blocked_surface_key,
+        sst: sst_payload
+      )
+    end
+
+    private
+
+    attr_reader :response, :url
+
+    def feed_assessment
+      Assessment.new(
+        surface_category: :unsupported_surface,
+        articles_count: 0,
+        admission_drops: {},
+        html_response: false
+      )
+    end
+
+    def html_parsed_body
+      return unless response.html_response?
+
+      response.parsed_body
+    rescue RequestService::UnsupportedResponseContentType
+      nil
+    end
+
+    def surface_category(parsed)
+      return :unsupported_surface unless parsed
+
+      AutoSource::Scraper.classify_no_scraper_surface(parsed, body: response.body)
+    end
+
+    def cheap_articles
+      return [0, {}] unless response.html_response?
+
+      source = AutoSource.new(response, AutoSource::DEFAULT_CONFIG.merge(limit: 10))
+      [source.articles.size, source.admission_drops]
+    end
+
+    def sst_payload_and_segments(requested)
+      return [nil, nil] unless response.html_response?
+
+      sst = sst_document
+      return [nil, nil] unless sst
+
+      stats = segment_stats(sst, requested)
+      [
+        { node_count: sst.node_count, degraded: sst.degraded, segment_stats: stats },
+        stats
+      ]
+    end
+
+    def sst_document
+      Html2rss::SST::Normalizer.call(response.body)
+    rescue ArgumentError
+      nil
+    end
+
+    def segment_stats(sst, page_url)
+      segments = self.class.discover_segments(sst, page_url)
+      return { found: 0 } if segments.empty?
+
+      {
+        found: segments.size,
+        strategies: segments.map(&:strategy).uniq,
+        sample_paths: segments.first(5).map { |s| s.root_node.tag_path }
+      }
+    end
+
+    def scheme_downgrade?(requested, final)
+      requested.scheme == 'https' && final.scheme == 'http'
+    end
+
+    def alternate_feeds_from(parsed)
+      return [] unless parsed.is_a?(Nokogiri::HTML::Document)
+
+      Html::FeedLink.from_document(parsed).map { |link| { href: link.href, mime_type: link.mime_type } }
+    end
+
+    def blocked_surface_key
+      blocked = RequestService::BlockedSurface.interstitial_signature_for(response.body)
+      blocked[:key].to_s if blocked
+    end
+  end
+end

data/lib/html2rss/recon.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require 'digest'
+require 'uri'
+require 'fileutils'
+
+module Html2rss
+  ##
+  # Service that runs reconnaissance on a URL to discover redirect chains,
+  # native RSS/Atom feeds, surface categorization, and emit an actionable verdict.
+  module Recon # rubocop:disable Metrics/ModuleLength
+    module_function
+
+    Verdict = Data.define(:name)
+
+    ##
+    # Closed curation verdict (:build / :defer / :drop).
+    class Verdict
+      # Closed set of curation verdict wire names.
+      NAMES = Set[:build, :defer, :drop].freeze
+
+      class << self
+        ##
+        # @param value [Verdict, Symbol, String]
+        # @return [Verdict]
+        def coerce(value)
+          return value if value.is_a?(self)
+
+          new(name: value.to_sym)
+        end
+      end
+
+      ##
+      # @param name [Symbol]
+      def initialize(name:)
+        raise ArgumentError, "unknown verdict: #{name.inspect}" unless NAMES.include?(name)
+
+        super
+      end
+
+      ##
+      # @return [Boolean]
+      def build? = name == :build
+
+      ##
+      # @return [Boolean]
+      def defer? = name == :defer
+
+      ##
+      # @return [Boolean]
+      def drop? = name == :drop
+
+      ##
+      # @return [Symbol]
+      def to_sym = name
+
+      ##
+      # @return [String]
+      def to_s = name.to_s
+    end
+
+    ##
+    # Immutable outcome of a reconnaissance operation.
+    Result = Data.define(
+      :requested_url,
+      :final_url,
+      :status,
+      :verdict,
+      :native_feed,
+      :surface_category,
+      :articles_count,
+      :scheme_downgrade,
+      :notes,
+      :html_bytesize
+    ) do
+      ##
+      # @return [Boolean]
+      def build? = verdict.build?
+
+      ##
+      # @return [Boolean]
+      def defer? = verdict.defer?
+
+      ##
+      # @return [Boolean]
+      def drop? = verdict.drop?
+
+      ##
+      # @return [Boolean]
+      def native_feed? = !native_feed.nil?
+
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h # rubocop:disable Metrics/MethodLength
+        {
+          requested_url: requested_url.to_s,
+          final_url: final_url.to_s,
+          status:,
+          verdict: verdict.to_sym,
+          native_feed: native_feed&.to_s,
+          surface_category: surface_category&.to_s,
+          articles_count:,
+          scheme_downgrade:,
+          notes:,
+          html_bytesize:
+        }.compact
+      end
+    end
+
+    ##
+    # Runs reconnaissance on a single URL.
+    #
+    # @param url [String, Html2rss::Url] source page URL
+    # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+    # @param cache_dir [String, nil] optional directory to cache raw HTML bodies
+    # @param cache_mutex [Mutex, nil] optional mutex serializing cache writes
+    # @option options [Integer, nil] :max_redirects optional maximum redirects
+    # @option options [Integer, nil] :max_requests optional request budget
+    # @return [Html2rss::Recon::Result]
+    def call(url, strategy: :auto, cache_dir: nil, cache_mutex: nil, **) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
+      url_obj = Url.from_absolute(url)
+
+      begin
+        probe = PageRecon.probe(url_obj, strategy:, **)
+      rescue StandardError => error
+        return error_result(url_obj, error)
+      end
+
+      cache_html_body(probe.response.body, url_obj, cache_dir, cache_mutex) if cache_dir && probe.response.body
+
+      recon = probe.result
+      native_feed, discovery_note = find_native_feed(url_obj, probe.session, probe.response)
+      notes = build_notes(recon, native_feed, probe.response)
+      notes << discovery_note if discovery_note
+      verdict = determine_verdict(recon, native_feed)
+
+      Result.new(
+        requested_url: url_obj,
+        final_url: probe.response.url,
+        status: probe.response.status,
+        verdict:,
+        native_feed:,
+        surface_category: SurfaceCategory.coerce(recon.surface_category),
+        articles_count: recon.articles_count,
+        scheme_downgrade: recon.scheme_downgrade,
+        notes:,
+        html_bytesize: probe.response.body&.bytesize
+      )
+    end
+
+    ##
+    # Runs batch reconnaissance across an Enumerable of URLs.
+    # Results are returned in input order. Concurrency uses a Thread pool
+    # (not Ractors) for I/O overlap with one loaded gem image.
+    #
+    # @param urls [Enumerable<String>] list of URLs
+    # @param strategy [Symbol] request strategy
+    # @param cache_dir [String, nil] optional directory to cache raw HTML bodies
+    # @param max_threads [Integer] concurrent worker count
+    # @option options [Integer, nil] :max_redirects optional maximum redirects
+    # @option options [Integer, nil] :max_requests optional request budget
+    # @return [Array<Html2rss::Recon::Result>]
+    def batch(urls, strategy: :auto, cache_dir: nil, max_threads: 5, **options)
+      url_list = urls.map { |u| u.to_s.strip }.reject(&:empty?)
+      return [] if url_list.empty?
+
+      FileUtils.mkdir_p(cache_dir) if cache_dir
+      cache_mutex = Mutex.new if cache_dir
+
+      Batch.map(url_list, concurrency: max_threads) do |target_url|
+        call(target_url, strategy:, cache_dir:, cache_mutex:, **options)
+      end
+    end
+
+    ##
+    # @param url_obj [Html2rss::Url]
+    # @param session [Html2rss::RequestSession]
+    # @param response [Html2rss::RequestService::Response]
+    # @return [Array(Html2rss::Url, nil), String, nil)] feed URL and optional discovery error note
+    def find_native_feed(url_obj, session, response)
+      feed = Syndication::Discovery.best_feed_url(
+        page_url: url_obj,
+        request_session: session,
+        parsed_body: (response.parsed_body if response.html_response?),
+        html: response.body
+      )
+      [feed, nil]
+    rescue StandardError => error
+      [nil, "discovery_error=#{error.class}: #{error.message}"]
+    end
+    private_class_method :find_native_feed
+
+    def build_notes(recon, native_feed, response)
+      notes = []
+      notes << "native_rss=#{native_feed}" if native_feed
+      if recon.scheme_downgrade
+        notes << 'scheme_downgrade'
+        notes << 'botasaurus_retry=Try strategy botasaurus once before DROP (HTTPS→HTTP may need JS fetch)'
+      end
+      notes << "blocked=#{recon.blocked_surface}" if recon.blocked_surface
+      notes << "html_bytes=#{response.body&.bytesize}" if response.body
+      notes
+    end
+    private_class_method :build_notes
+
+    def determine_verdict(recon, native_feed)
+      return Verdict.coerce(:drop) if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
+      return Verdict.coerce(:defer) if native_feed
+
+      category = SurfaceCategory.coerce(recon.surface_category)
+      return Verdict.coerce(:drop) if category.blocked?
+
+      Verdict.coerce(:build)
+    end
+    private_class_method :determine_verdict
+
+    def error_result(url_obj, error) # rubocop:disable Metrics/MethodLength
+      Result.new(
+        requested_url: url_obj,
+        final_url: url_obj,
+        status: nil,
+        verdict: Verdict.coerce(:drop),
+        native_feed: nil,
+        surface_category: SurfaceCategory.coerce(:unsupported_surface),
+        articles_count: 0,
+        scheme_downgrade: false,
+        notes: ["error: #{error.class} - #{error.message}"],
+        html_bytesize: nil
+      )
+    end
+    private_class_method :error_result
+
+    # Writes the raw response body once under a host+url-digest filename.
+    #
+    # @param body [String]
+    # @param url_obj [Html2rss::Url]
+    # @param cache_dir [String]
+    # @param cache_mutex [Mutex, nil]
+    # @return [void]
+    def cache_html_body(body, url_obj, cache_dir, cache_mutex)
+      host = url_obj.host.to_s.delete_prefix('www.')
+      host = 'snapshot' if host.empty?
+      digest = Digest::SHA256.hexdigest(url_obj.to_s)[0, 12]
+      file_path = File.join(cache_dir, "#{host}-#{digest}.html")
+      writer = -> { File.write(file_path, body) }
+      cache_mutex ? cache_mutex.synchronize(&writer) : writer.call
+    rescue StandardError => error
+      Log.debug("Recon cache failed: #{error.message}")
+    end
+    private_class_method :cache_html_body
+  end
+end

data/lib/html2rss/request_service/blocked_surface.rb

@@ -18,6 +18,7 @@ module Html2rss
             /checking your browser before accessing/i,
             /please (?:enable|turn on) javascript and cookies/i,
             %r{cdn-cgi/challenge-platform}i,
+            /cf-challenge/i,
             /cloudflare ray id/i
           ],
           message: 'Blocked surface detected: Cloudflare anti-bot interstitial page. ' \

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -43,7 +43,7 @@ module Html2rss
       MAX_RETRIES = 3
 
       # Allowlisted ScrapeDiagnostics keys nested under diagnostics.
-      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge].freeze
+      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge timeout_phase].freeze
       # Allowlisted ChallengeSignal keys nested under diagnostics.challenge.
       CHALLENGE_KEYS = %w[blocked detected marker].freeze
 
@@ -185,10 +185,17 @@ module Html2rss
             'Botasaurus challenge block detected.'
         end
 
+        # @return [String, nil] scrape-api timeout stage when present on diagnostics
+        def timeout_phase
+          value = diagnostics['timeout_phase']
+          value.is_a?(String) && !value.empty? ? value : nil
+        end
+
         # @return [String] actionable upstream failure summary
         def failure_message
           details = ["status=#{transport_status}", "error_category=#{error_category}", "error=#{error}"]
           details << "request_id=#{request_id}" if request_id
+          details << "timeout_phase=#{timeout_phase}" if timeout_phase
           "Botasaurus scrape failed (#{details.join(', ')})."
         end
 

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -59,7 +59,7 @@ module Html2rss
         return unless error.timeout?
 
         log_timeout!(reason: 'botasaurus_upstream')
-        raise RequestTimedOut, error.failure_message
+        raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
       end
 
       def response_url(final_url)

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -10,7 +10,7 @@ module Html2rss
     ##
     # Strategy to use Faraday for the request.
     # @see https://rubygems.org/gems/faraday
-    # rubocop:disable Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
+    # rubocop:disable-next Metrics/ClassLength -- terminal redirect retry colocated with Faraday transport
     class FaradayStrategy < Strategy
       ##
       # Restores buffered streamed bytes so response middleware can process them.
@@ -59,9 +59,13 @@ module Html2rss
       def request_with_terminal_redirect_retry(response_guard, deadline:)
         faraday_request(response_guard, deadline:, streaming_buffer: true)
       rescue Faraday::FollowRedirects::RedirectLimitReached => error
-        raise error unless terminal_redirect_retryable?
+        raise RedirectLimitReached, error.message unless terminal_redirect_retryable?
 
-        retry_from_terminal_redirect!(response_guard, deadline:)
+        begin
+          retry_from_terminal_redirect!(response_guard, deadline:)
+        rescue Faraday::FollowRedirects::RedirectLimitReached => retry_error
+          raise RedirectLimitReached, retry_error.message
+        end
       end
 
       def terminal_redirect_retryable?
@@ -100,7 +104,9 @@ module Html2rss
 
         client.get do |req|
           apply_timeouts(req, deadline:)
-          buffer = prepare_stream_buffer(req) if streaming_buffer
+          next unless streaming_buffer
+
+          buffer = prepare_stream_buffer(req)
           req.options.on_data = on_data_callback(response_guard, buffer)
         end
       end
@@ -146,7 +152,7 @@ module Html2rss
         end
       end
 
-      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable-next Metrics/AbcSize
       def client
         @client ||= Faraday.new(url: request_url.to_s, headers: ctx.headers) do |faraday|
           faraday.use Faraday::FollowRedirects::Middleware, limit: ctx.policy.max_redirects, callback: redirect_callback
@@ -157,7 +163,6 @@ module Html2rss
           end
         end
       end
-      # rubocop:enable Metrics/AbcSize
 
       def apply_timeouts(request, deadline:)
         remaining_timeout = remaining_timeout_seconds(deadline)
@@ -224,6 +229,5 @@ module Html2rss
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       end
     end
-    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/html2rss/request_service/policy.rb

@@ -84,11 +84,10 @@ module Html2rss
       # Returns the default request policy.
       #
       # @return [Policy] a default, frozen policy instance
-      # rubocop:disable Layout/ClassStructure
+      # rubocop:disable-next Layout/ClassStructure
       def self.default
         new
       end
-      # rubocop:enable Layout/ClassStructure
 
       ##
       # Validates whether a request target is permitted for the given context.