html2rss 0.27.1 → 0.28.0

data/lib/html2rss/page_recon.rb

@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Shared page recon for MCP Inspect and FeedResolution probes.
+  #
+  # Owns surface class, native feed hints, segment stats, and a cheap AutoSource
+  # article count — not request strategy selection or MCP next-step policy.
+  class PageRecon # rubocop:disable Metrics/ClassLength -- recon bag stays co-located
+    ##
+    # Cheap surface + admission facts shared by AutoFallback gates and FeedResolution probes.
+    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response) do
+      ##
+      # @return [Html2rss::SurfaceCategory]
+      def category = SurfaceCategory.coerce(surface_category)
+
+      ##
+      # @return [Boolean]
+      def weak? = category.weak?
+
+      ##
+      # @return [Boolean]
+      def blocked? = category.blocked?
+
+      ##
+      # @return [Boolean]
+      def listing_bonus? = category.listing_bonus?
+    end
+    ##
+    # Recon facts used by Inspect and FeedResolution.
+    Result = Data.define(
+      :requested_url,
+      :final_url,
+      :status,
+      :scheme_downgrade,
+      :alternate_feeds,
+      :surface_category,
+      :articles_count,
+      :admission_drops,
+      :segment_stats,
+      :html_response,
+      :content_type,
+      :blocked_surface,
+      :sst
+    ) do
+      ##
+      # @return [Hash{Symbol => Object}]
+      def to_h # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- omit-empty optional keys
+        {
+          requested_url:,
+          final_url:,
+          status:,
+          scheme_downgrade:,
+          alternate_feeds:,
+          surface_category:,
+          articles_count:,
+          html_response:,
+          content_type:,
+          **(admission_drops.any? ? { admission_drops: } : {}),
+          **(segment_stats ? { segment_stats: } : {}),
+          **(blocked_surface ? { blocked_surface: } : {}),
+          **(sst ? { sst: } : {})
+        }
+      end
+    end
+
+    ##
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url] requested entry URL
+    # @param strategy [Symbol, nil] unused (reserved for callers that already chose a strategy)
+    # @return [Result]
+    def self.call(response:, url:, strategy: nil) # rubocop:disable Lint/UnusedMethodArgument
+      new(response:, url:).call
+    end
+
+    ##
+    # Cheap page assessment for policy gates and probe scoring (fixed AutoSource limit).
+    #
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    # @return [Assessment]
+    def self.assess(response:, url:)
+      new(response:, url:).assess
+    end
+
+    ##
+    # Surface class only — no AutoSource extract (for empty-extract error labels).
+    #
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    # @return [Symbol]
+    def self.surface_category_for(response:, url:)
+      new(response:, url:).surface_category_for
+    end
+
+    ##
+    # @param sst [Html2rss::SST::Document]
+    # @param url [String, Html2rss::Url]
+    # @return [Array]
+    def self.discover_segments(sst, url)
+      link_resolver = Scoring::LinkResolver.new(url)
+      AutoSource::Segmenter.call(sst, base_url: url, strategy: :list, link_resolver:)
+    rescue StandardError
+      []
+    end
+
+    ##
+    # @param response [Html2rss::RequestService::Response]
+    # @param url [String, Html2rss::Url]
+    def initialize(response:, url:)
+      @response = response
+      @url = url
+    end
+
+    ##
+    # @return [Assessment]
+    def assess
+      return feed_assessment if response.feed_response?
+
+      parsed = html_parsed_body
+      articles_count, admission_drops = cheap_articles
+      Assessment.new(
+        surface_category: surface_category(parsed),
+        articles_count:,
+        admission_drops:,
+        html_response: true
+      )
+    end
+
+    ##
+    # @return [Symbol]
+    def surface_category_for
+      return :unsupported_surface if response.feed_response?
+
+      surface_category(html_parsed_body)
+    end
+
+    ##
+    # @return [Result]
+    def call # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- assemble recon Result
+      requested = Url.from_absolute(url)
+      final = response.url
+      parsed = html_parsed_body
+      assessment = assess
+      sst_payload, segment_stats = sst_payload_and_segments(requested)
+
+      Result.new(
+        requested_url: requested.to_s,
+        final_url: final.to_s,
+        status: response.status,
+        scheme_downgrade: scheme_downgrade?(requested, final),
+        alternate_feeds: alternate_feeds_from(parsed),
+        surface_category: assessment.surface_category,
+        articles_count: assessment.articles_count,
+        admission_drops: assessment.admission_drops,
+        segment_stats:,
+        html_response: response.html_response?,
+        content_type: response.content_type,
+        blocked_surface: blocked_surface_key,
+        sst: sst_payload
+      )
+    end
+
+    private
+
+    attr_reader :response, :url
+
+    def feed_assessment
+      Assessment.new(
+        surface_category: :unsupported_surface,
+        articles_count: 0,
+        admission_drops: {},
+        html_response: false
+      )
+    end
+
+    def html_parsed_body
+      return unless response.html_response?
+
+      response.parsed_body
+    rescue RequestService::UnsupportedResponseContentType
+      nil
+    end
+
+    def surface_category(parsed)
+      return :unsupported_surface unless parsed
+
+      AutoSource::Scraper.classify_no_scraper_surface(parsed, body: response.body)
+    end
+
+    def cheap_articles
+      return [0, {}] unless response.html_response?
+
+      source = AutoSource.new(response, AutoSource::DEFAULT_CONFIG.merge(limit: 10))
+      [source.articles.size, source.admission_drops]
+    end
+
+    def sst_payload_and_segments(requested)
+      return [nil, nil] unless response.html_response?
+
+      sst = sst_document
+      return [nil, nil] unless sst
+
+      stats = segment_stats(sst, requested)
+      [
+        { node_count: sst.node_count, degraded: sst.degraded, segment_stats: stats },
+        stats
+      ]
+    end
+
+    def sst_document
+      Html2rss::SST::Normalizer.call(response.body)
+    rescue ArgumentError
+      nil
+    end
+
+    def segment_stats(sst, page_url)
+      segments = self.class.discover_segments(sst, page_url)
+      return { found: 0 } if segments.empty?
+
+      {
+        found: segments.size,
+        strategies: segments.map(&:strategy).uniq,
+        sample_paths: segments.first(5).map { |s| s.root_node.tag_path }
+      }
+    end
+
+    def scheme_downgrade?(requested, final)
+      requested.scheme == 'https' && final.scheme == 'http'
+    end
+
+    def alternate_feeds_from(parsed)
+      return [] unless parsed.is_a?(Nokogiri::HTML::Document)
+
+      Html::FeedLink.from_document(parsed).map { |link| { href: link.href, mime_type: link.mime_type } }
+    end
+
+    def blocked_surface_key
+      blocked = RequestService::BlockedSurface.interstitial_signature_for(response.body)
+      blocked[:key].to_s if blocked
+    end
+  end
+end

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -7,7 +7,7 @@ module Html2rss
     ##
     # Maps html2rss request/response handling to botasaurus-scrape-api OpenAPI 2.0
     # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
-    class BotasaurusContract # rubocop:disable Metrics/ClassLength -- Success/Error envelopes stay with the owner
+    class BotasaurusContract
       # Closed set from OpenAPI ExecutionMode.
       EXECUTION_MODES = %w[auto request browser].freeze
       # Closed set from OpenAPI NavigationMode.
@@ -25,27 +25,28 @@ module Html2rss
       # OpenAPI WindowSize required keys (positive integers).
       WINDOW_SIZE_PROPERTIES = %i[width height].freeze
 
-      # Published wait clamp floor (`[1, 20]` in OpenAPI description).
+      # Faraday POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
+      SCRAPE_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_TIMEOUT_SECONDS', 45))
+      # Post-boot navigate/wait budget mirrored from scrape-api (`SCRAPE_WORK_TIMEOUT_SECONDS`).
+      SCRAPE_WORK_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS', 30))
+      # Seconds added to the scrape total for client transport overhead.
+      TRANSPORT_BUFFER_SECONDS = 2
+
+      # Published wait clamp floor (`[1, SCRAPE_WORK_TIMEOUT_SECONDS]` in OpenAPI description).
       MIN_WAIT_TIMEOUT_SECONDS = 1
       # Published wait clamp ceiling; YAML values above this are rejected.
-      MAX_WAIT_TIMEOUT_SECONDS = 20
+      MAX_WAIT_TIMEOUT_SECONDS = SCRAPE_WORK_TIMEOUT_SECONDS
       # OpenAPI wait_timeout_seconds default (omitted from the client payload).
-      DEFAULT_WAIT_TIMEOUT_SECONDS = 15
+      DEFAULT_WAIT_TIMEOUT_SECONDS = [15, SCRAPE_WORK_TIMEOUT_SECONDS].min
 
       # OpenAPI max_retries maximum (default 2 is applied upstream when omitted).
       MAX_RETRIES = 3
 
       # Allowlisted ScrapeDiagnostics keys nested under diagnostics.
-      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge].freeze
+      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge timeout_phase].freeze
       # Allowlisted ChallengeSignal keys nested under diagnostics.challenge.
       CHALLENGE_KEYS = %w[blocked detected marker].freeze
 
-      # Remaining seconds at or below which Botasaurus retries are disabled.
-      TIGHT_BUDGET_SECONDS = 12
-
-      # Seconds reserved from remaining budget before setting wait_timeout_seconds.
-      BUDGET_WAIT_RESERVE_SECONDS = 2
-
       ##
       # Parsed OpenAPI ScrapeSuccess envelope (HTTP 200).
       class Success
@@ -184,10 +185,17 @@ module Html2rss
             'Botasaurus challenge block detected.'
         end
 
+        # @return [String, nil] scrape-api timeout stage when present on diagnostics
+        def timeout_phase
+          value = diagnostics['timeout_phase']
+          value.is_a?(String) && !value.empty? ? value : nil
+        end
+
         # @return [String] actionable upstream failure summary
         def failure_message
           details = ["status=#{transport_status}", "error_category=#{error_category}", "error=#{error}"]
           details << "request_id=#{request_id}" if request_id
+          details << "timeout_phase=#{timeout_phase}" if timeout_phase
           "Botasaurus scrape failed (#{details.join(', ')})."
         end
 
@@ -246,7 +254,6 @@ module Html2rss
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param headers [Hash] request headers from context
       # @param options [Hash] validated request.botasaurus options
-      # @param remaining_timeout_seconds [Numeric, nil] shared request budget remainder for clamps
       # @option options [String] :execution_mode
       # @option options [String] :navigation_mode
       # @option options [Integer] :max_retries
@@ -264,19 +271,19 @@ module Html2rss
       # @option options [String] :lang
       # @option options [Hash] :cookies
       # @option options [Hash] :headers
-      def initialize(url:, headers: {}, options: {}, remaining_timeout_seconds: nil)
+      def initialize(url:, headers: {}, options: {})
         @url = url
         @headers = headers
         @options = options
-        @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape (explicit options plus budget clamp-downs)
+      # @return [Hash] payload for POST /scrape (explicit options plus wait cap)
       def request_payload
         payload = { url: url.to_s }.merge(filtered_options)
         forwarded_headers = merged_headers
         payload[:headers] = forwarded_headers if forwarded_headers&.any?
-        clamp_for_budget(payload)
+        cap_wait_timeout!(payload)
+        payload
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
@@ -291,7 +298,7 @@ module Html2rss
 
       private
 
-      attr_reader :url, :headers, :options, :remaining_timeout_seconds
+      attr_reader :url, :headers, :options
 
       def json_object(body)
         payload = JSON.parse(body)
@@ -315,36 +322,6 @@ module Html2rss
         end
       end
 
-      def clamp_for_budget(payload)
-        remaining = remaining_timeout_seconds
-        clamped = payload.dup
-        unless remaining.nil?
-          apply_tight_retries!(clamped, remaining)
-          apply_budget_wait!(clamped, remaining)
-        end
-        cap_wait_timeout!(clamped)
-        clamped
-      end
-
-      def apply_tight_retries!(payload, remaining)
-        payload[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
-      end
-
-      def apply_budget_wait!(payload, remaining)
-        budget_wait = budget_wait_seconds(remaining)
-        configured = payload[:wait_timeout_seconds]
-        if configured
-          payload[:wait_timeout_seconds] = [configured, budget_wait].min
-        elsif budget_wait < DEFAULT_WAIT_TIMEOUT_SECONDS
-          payload[:wait_timeout_seconds] = budget_wait
-        end
-      end
-
-      def budget_wait_seconds(remaining)
-        reserved = [MIN_WAIT_TIMEOUT_SECONDS, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
-        [reserved, MAX_WAIT_TIMEOUT_SECONDS].min
-      end
-
       def cap_wait_timeout!(payload)
         wait = payload[:wait_timeout_seconds]
         return unless wait

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -2,12 +2,20 @@
 
 require 'faraday'
 require 'json'
+require 'securerandom'
 
 module Html2rss
   class RequestService
     ##
     # Strategy to delegate fetching to a Botasaurus scrape API.
     class BotasaurusStrategy < Strategy
+      # Content-Type negotiation for the scrape API transport hop.
+      TRANSPORT_ACCEPT = 'application/json'
+      # Disable compressed bodies so Faraday returns raw JSON without implicit decoding surprises.
+      TRANSPORT_ENCODING = 'identity'
+      # Correlates each POST /scrape with botasaurus-scrape-api request logs.
+      REQUEST_ID_HEADER = 'X-Request-Id'
+
       private
 
       def fetch
@@ -18,7 +26,9 @@ module Html2rss
       end
 
       def post_scrape_request
-        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), content_type_header)
+        request_id = SecureRandom.uuid
+        Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
+        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), post_headers(request_id))
         contract.parse_response(transport_response)
       end
 
@@ -49,7 +59,7 @@ module Html2rss
         return unless error.timeout?
 
         log_timeout!(reason: 'botasaurus_upstream')
-        raise RequestTimedOut, error.failure_message
+        raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
       end
 
       def response_url(final_url)
@@ -64,13 +74,28 @@ module Html2rss
         @contract ||= BotasaurusContract.new(
           url: ctx.url,
           headers: ctx.headers,
-          options: ctx.request.fetch(:botasaurus, {}),
-          remaining_timeout_seconds: attempt_timeout_seconds
+          options: ctx.request.fetch(:botasaurus, {})
         )
       end
 
       def client
-        @client ||= Faraday.new(url: scraper_base_url.to_s, request: request_options)
+        # No :gzip middleware on this client — compression is for remote target fetches only.
+        @client ||= Faraday.new(url: scraper_base_url.to_s, headers: client_headers, request: request_options)
+      end
+
+      def client_headers
+        {
+          'User-Agent' => Config::RequestHeaders::DEFAULT_USER_AGENT,
+          'Accept' => TRANSPORT_ACCEPT,
+          'Accept-Encoding' => TRANSPORT_ENCODING
+        }
+      end
+
+      def post_headers(request_id)
+        {
+          'Content-Type' => 'application/json',
+          REQUEST_ID_HEADER => request_id
+        }
       end
 
       def request_options
@@ -78,13 +103,10 @@ module Html2rss
       end
 
       def attempt_timeout_seconds
-        @attempt_timeout_seconds ||= ctx.budget.effective_timeout_seconds(
-          fallback: ctx.policy.total_timeout_seconds
-        )
-      end
-
-      def content_type_header
-        { 'Content-Type' => 'application/json' }
+        @attempt_timeout_seconds ||= begin
+          budget = ctx.budget.effective_timeout_seconds(fallback: ctx.policy.total_timeout_seconds)
+          [budget, BotasaurusContract::SCRAPE_TIMEOUT_SECONDS + BotasaurusContract::TRANSPORT_BUFFER_SECONDS].min
+        end
       end
 
       def scraper_base_url

data/lib/html2rss/request_service/response.rb

@@ -14,11 +14,15 @@ module Html2rss
 
       # Bodies that look like HTML even when Content-Type is missing, empty, or wrong.
       HTML_BODY_SNIFF = /\A\s*(?:<!DOCTYPE\s+html|<html)/i
+      # Content-Type markers for RSS/Atom syndication responses.
+      FEED_CT_MARKERS = %w[rss+xml atom+xml rss atom].freeze
+      # Body sniff for unlabeled syndication (first 800 bytes, downcased).
+      FEED_BODY_MARKERS = ['<rss', '<feed xmlns', '<feed '].freeze
       # Charset from Content-Type or a leading <meta charset>.
       CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
       # Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
       META_CHARSET_BYTES = 2048
-      private_constant :CHARSET_PARAMETER, :META_CHARSET_BYTES
+      private_constant :CHARSET_PARAMETER, :META_CHARSET_BYTES, :FEED_CT_MARKERS, :FEED_BODY_MARKERS
 
       ##
       # @param body [String] the body of the response
@@ -74,7 +78,14 @@ module Html2rss
 
       # @return [Boolean] whether response content is HTML (header or sniffed body, never JSON)
       def html_response?
-        content_type.include?('text/html') || (!json_response? && html_looking_body?)
+        content_type.include?('text/html') || (!json_response? && !feed_response? && html_looking_body?)
+      end
+
+      # @return [Boolean] whether response content is RSS/Atom (header or sniffed body)
+      def feed_response?
+        return @feed_response if defined?(@feed_response)
+
+        @feed_response = feed_content_type? || (!json_response? && !html_looking_body? && feed_looking_body?)
       end
 
       ##
@@ -143,6 +154,19 @@ module Html2rss
       def html_looking_body?
         body.to_s.b.match?(HTML_BODY_SNIFF)
       end
+
+      def feed_content_type?
+        ct = content_type.downcase
+        return false if ct.empty? || ct.include?('html') || ct.include?('json')
+
+        # Substring match against Content-Type (not Array#intersect? on a String).
+        FEED_CT_MARKERS.any? { |marker| ct.include?(marker) } # rubocop:disable Style/ArrayIntersect
+      end
+
+      def feed_looking_body?
+        snippet = body.to_s[0, 800].downcase
+        FEED_BODY_MARKERS.any? { |marker| snippet.include?(marker) } # rubocop:disable Style/ArrayIntersect
+      end
     end
   end
 end

data/lib/html2rss/request_service.rb

@@ -31,8 +31,20 @@ module Html2rss
     class ResponseTooLarge < Html2rss::Error; end
     # Raised when blocked content surfaces are detected.
     class BlockedSurfaceDetected < Html2rss::Error; end
+
     # Raised when a request times out.
-    class RequestTimedOut < Html2rss::Error; end
+    class RequestTimedOut < Html2rss::Error
+      ##
+      # @param message [String, nil] timeout failure summary
+      # @param timeout_phase [String, nil] scrape-api stage when known (+queue+/+boot+/+work+)
+      def initialize(message = nil, timeout_phase: nil)
+        @timeout_phase = timeout_phase
+        super(message)
+      end
+
+      # @return [String, nil] scrape-api timeout stage, or nil for transport/budget timeouts
+      attr_reader :timeout_phase
+    end
 
     # Raised when Botasaurus configuration is missing or invalid.
     class BotasaurusConfigurationError < Html2rss::Error

data/lib/html2rss/request_session.rb

@@ -15,14 +15,17 @@ module Html2rss
       # @param strategy [Symbol] request strategy for the session
       # @param budget [RequestService::Budget] shared request budget
       # @param policy [RequestService::Policy] request policy (from FeedPipeline::RuntimePolicy.resources_for)
+      # @param scrape_url [String, nil] effective fetch URL when it differs from config channel URL
       # @param logger [Logger] logger used for operational warnings
       # @return [RequestSession] configured request session
-      def build(config:, strategy:, budget:, policy:, logger: Html2rss::Log)
+      # rubocop:disable Metrics/ParameterLists -- scrape_url override stays beside config
+      def build(config:, strategy:, budget:, policy:, scrape_url: nil, logger: Html2rss::Log)
         context = RequestService::Context.new(
-          url: config.url, headers: config.headers, request: config.request, policy:, budget:
+          url: scrape_url || config.url, headers: config.headers, request: config.request, policy:, budget:
         )
         new(context:, strategy:, logger:)
       end
+      # rubocop:enable Metrics/ParameterLists
     end
 
     ##

data/lib/html2rss/scrape_target.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Html2rss
+  ##
+  # Immutable entry vs effective scrape URLs for one pipeline run.
+  #
+  # Replaces mutating {Config#scrape_url=} after {FeedResolution} rewrites the fetch URL.
+  ScrapeTarget = Data.define(:entry_url, :effective_url) do
+    ##
+    # @param config [Html2rss::Config]
+    # @return [ScrapeTarget]
+    def self.from_config(config)
+      entry = config.url
+      new(entry_url: entry, effective_url: entry)
+    end
+
+    ##
+    # @param url [String, Html2rss::Url]
+    # @return [ScrapeTarget]
+    def with_effective(url)
+      self.class.new(entry_url:, effective_url: Url.from_absolute(url).to_s)
+    end
+  end
+end