html2rss 0.29.1 → 0.31.0

data/lib/html2rss/page_recon/diagnostics.rb

@@ -6,9 +6,11 @@ module Html2rss
     # Diagnostic inspect path (not Capture or Recon ownership). Fetches via {.probe},
     # then adds scraper/XHR diagnostics for curation inspect surfaces.
     module Diagnostics # rubocop:disable Metrics/ModuleLength -- diagnostic wire fields stay co-located
+      Report = Data.define(:data)
+
       ##
       # Typed diagnostic report for inspect wire payloads and Outcome policy.
-      Report = Data.define(:data) do
+      class Report
         ##
         # @return [Boolean]
         def alternate_feeds?
@@ -115,97 +117,85 @@ module Html2rss
         end
       end
 
-      def build_data(probe, recon, response)
-        data = recon.to_h.merge(
-          strategy: probe.strategy,
-          scraper_eligibility: scraper_info(safe_parsed_body(response)),
-          html_present: html_present?(recon, response),
-          likely_js_shell: likely_js_shell?(recon, response),
-          redirect_summary: redirect_summary(recon)
-        )
-        data[:xhr_capture] = xhr_capture_info(response) if probe.strategy == :botasaurus
-        log_js_shell(data, response.body&.bytesize.to_i) if data[:likely_js_shell]
-        data
-      end
-      module_function :build_data
-      private_class_method :build_data
+      class << self
+        private
+
+        def build_data(probe, recon, response)
+          data = recon.to_h.merge(
+            strategy: probe.strategy,
+            scraper_eligibility: scraper_info(safe_parsed_body(response)),
+            html_present: html_present?(recon, response),
+            likely_js_shell: likely_js_shell?(recon, response),
+            redirect_summary: redirect_summary(recon)
+          )
+          data[:xhr_capture] = xhr_capture_info(response) if probe.strategy == :botasaurus
+          log_js_shell(data, response.body&.bytesize.to_i) if data[:likely_js_shell]
+          data
+        end
 
-      def resolve_inspect_strategy(strategy, deep:)
-        name = (strategy || :auto).to_sym
-        return :botasaurus if deep && name == :auto && MCP::Runtime.botasaurus_configured?
+        def resolve_inspect_strategy(strategy, deep:)
+          name = (strategy || :auto).to_sym
+          return :botasaurus if deep && name == :auto && MCP::Runtime.botasaurus_configured?
 
-        name
-      end
-      module_function :resolve_inspect_strategy
-      private_class_method :resolve_inspect_strategy
+          name
+        end
 
-      def html_present?(recon, response)
-        recon.html_response && !response.body.to_s.empty?
-      end
-      module_function :html_present?
-      private_class_method :html_present?
+        def html_present?(recon, response)
+          recon.html_response && !response.body.to_s.empty?
+        end
 
-      def likely_js_shell?(recon, response)
-        return false unless html_present?(recon, response)
-        return false if recon.articles_count.positive?
-        return false if recon.blocked_surface || recon.surface_category == :blocked_surface
+        def likely_js_shell?(recon, response)
+          return false unless html_present?(recon, response)
+          return false if recon.articles_count.positive?
+          return false if recon.blocked_surface || recon.surface_category == :blocked_surface
 
-        return true if recon.surface_category == :app_shell
+          return true if recon.surface_category == :app_shell
 
-        response.body.bytesize >= JS_SHELL_MIN_BODY_BYTES &&
-          SurfaceCategory.coerce(recon.surface_category).weak?
-      end
-      module_function :likely_js_shell?
-      private_class_method :likely_js_shell?
+          response.body.bytesize >= JS_SHELL_MIN_BODY_BYTES &&
+            SurfaceCategory[recon.surface_category].weak?
+        end
 
-      def redirect_summary(recon)
-        {
-          requested_url: recon.requested_url,
-          final_url: recon.final_url,
-          status: recon.status,
-          scheme_downgrade: recon.scheme_downgrade
-        }
-      end
-      module_function :redirect_summary
-      private_class_method :redirect_summary
+        def redirect_summary(recon)
+          {
+            requested_url: recon.requested_url,
+            final_url: recon.final_url,
+            status: recon.status,
+            scheme_downgrade: recon.scheme_downgrade
+          }
+        end
 
-      def log_js_shell(data, body_bytesize)
-        Log.debug(
-          "Diagnostics js_shell: bytesize=#{body_bytesize} surface_category=#{data[:surface_category]}"
-        )
-      end
-      module_function :log_js_shell
-      private_class_method :log_js_shell
+        def log_js_shell(data, body_bytesize)
+          Log.debug(
+            "Diagnostics js_shell: bytesize=#{body_bytesize} surface_category=#{data[:surface_category]}"
+          )
+        end
 
-      def safe_parsed_body(response)
-        return unless response.html_response?
+        def safe_parsed_body(response)
+          return unless response.html_response?
 
-        response.parsed_body
-      rescue RequestService::UnsupportedResponseContentType
-        nil
-      end
-      module_function :safe_parsed_body
-      private_class_method :safe_parsed_body
-
-      def error_report(url, error) # rubocop:disable Metrics/MethodLength -- error hash mirrors success report shape
-        Report.new(
-          data: {
-            requested_url: url.to_s,
-            final_url: url.to_s,
-            status: nil,
-            scheme_downgrade: false,
-            alternate_feeds: [],
-            surface_category: :unsupported_surface,
-            articles_count: 0,
-            html_response: false,
-            content_type: nil,
-            strategy: nil,
-            scraper_eligibility: { error: "#{error.class} - #{error.message}" }
-          }
-        )
+          response.parsed_body
+        rescue RequestService::UnsupportedResponseContentType
+          nil
+        end
+
+        def error_report(url, error) # rubocop:disable Metrics/MethodLength -- error hash mirrors success report shape
+          Report.new(
+            data: {
+              requested_url: url.to_s,
+              final_url: url.to_s,
+              status: nil,
+              scheme_downgrade: false,
+              alternate_feeds: [],
+              surface_category: :unsupported_surface,
+              articles_count: 0,
+              html_response: false,
+              content_type: nil,
+              strategy: nil,
+              scraper_eligibility: { error: "#{error.class} - #{error.message}" }
+            }
+          )
+        end
       end
-      module_function :error_report
-      private_class_method :error_report
     end
   end
 end

data/lib/html2rss/page_recon.rb

@@ -8,12 +8,14 @@ module Html2rss
   # article count. Diagnostic fetch for Inspect/Recon lives on {.probe} — not twin
   # fetch helpers in those callers.
   class PageRecon # rubocop:disable Metrics/ClassLength -- recon bag stays co-located
+    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response)
+
     ##
     # Cheap surface + admission facts shared by AutoFallback gates and FeedResolution probes.
-    Assessment = Data.define(:surface_category, :articles_count, :admission_drops, :html_response) do
+    class Assessment
       ##
       # @return [Html2rss::SurfaceCategory]
-      def category = SurfaceCategory.coerce(surface_category)
+      def category = SurfaceCategory[surface_category]
 
       ##
       # @return [Boolean]
@@ -27,8 +29,6 @@ module Html2rss
       # @return [Boolean]
       def listing_bonus? = category.listing_bonus?
     end
-    ##
-    # Recon facts used by Inspect and FeedResolution.
     Result = Data.define(
       :requested_url,
       :final_url,
@@ -43,7 +43,11 @@ module Html2rss
       :content_type,
       :blocked_surface,
       :sst
-    ) do
+    )
+
+    ##
+    # Recon facts used by Inspect and FeedResolution.
+    class Result
       ##
       # @return [Hash{Symbol => Object}]
       def to_h # rubocop:disable Metrics/AbcSize, Metrics/MethodLength -- omit-empty optional keys

data/lib/html2rss/recon.rb

@@ -16,27 +16,14 @@ module Html2rss
     ##
     # Closed curation verdict (:build / :defer / :drop).
     class Verdict
-      # Closed set of curation verdict wire names.
-      NAMES = Set[:build, :defer, :drop].freeze
-
-      class << self
-        ##
-        # @param value [Verdict, Symbol, String]
-        # @return [Verdict]
-        def coerce(value)
-          return value if value.is_a?(self)
-
-          new(name: value.to_sym)
-        end
-      end
+      # Frozen name → instance table.
+      ALL = %i[build defer drop].to_h { |name| [name, new(name:)] }.freeze
+      private_class_method :new
 
       ##
-      # @param name [Symbol]
-      def initialize(name:)
-        raise ArgumentError, "unknown verdict: #{name.inspect}" unless NAMES.include?(name)
-
-        super
-      end
+      # @param name [Symbol, String]
+      # @return [Verdict]
+      def self.[](name) = ALL.fetch(name.to_sym) { raise ArgumentError, "unknown verdict: #{name.inspect}" }
 
       ##
       # @return [Boolean]
@@ -59,8 +46,6 @@ module Html2rss
       def to_s = name.to_s
     end
 
-    ##
-    # Immutable outcome of a reconnaissance operation.
     Result = Data.define(
       :requested_url,
       :final_url,
@@ -72,7 +57,11 @@ module Html2rss
       :scheme_downgrade,
       :notes,
       :html_bytesize
-    ) do
+    )
+
+    ##
+    # Immutable outcome of a reconnaissance operation.
+    class Result
       ##
       # @return [Boolean]
       def build? = verdict.build?
@@ -111,7 +100,7 @@ module Html2rss
     # Runs reconnaissance on a single URL.
     #
     # @param url [String, Html2rss::Url] source page URL
-    # @param strategy [Symbol] request strategy (:auto, :faraday, :botasaurus)
+    # @param strategy [Symbol] request strategy (:auto, :default, :botasaurus)
     # @param cache_dir [String, nil] optional directory to cache raw HTML bodies
     # @param cache_mutex [Mutex, nil] optional mutex serializing cache writes
     # @option options [Integer, nil] :max_redirects optional maximum redirects
@@ -140,7 +129,7 @@ module Html2rss
         status: probe.response.status,
         verdict:,
         native_feed:,
-        surface_category: SurfaceCategory.coerce(recon.surface_category),
+        surface_category: SurfaceCategory[recon.surface_category],
         articles_count: recon.articles_count,
         scheme_downgrade: recon.scheme_downgrade,
         notes:,
@@ -204,13 +193,13 @@ module Html2rss
     private_class_method :build_notes
 
     def determine_verdict(recon, native_feed)
-      return Verdict.coerce(:drop) if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
-      return Verdict.coerce(:defer) if native_feed
+      return Verdict[:drop] if recon.status.nil? || recon.status >= 400 || recon.scheme_downgrade
+      return Verdict[:defer] if native_feed
 
-      category = SurfaceCategory.coerce(recon.surface_category)
-      return Verdict.coerce(:drop) if category.blocked?
+      category = SurfaceCategory[recon.surface_category]
+      return Verdict[:drop] if category.blocked?
 
-      Verdict.coerce(:build)
+      Verdict[:build]
     end
     private_class_method :determine_verdict
 
@@ -219,9 +208,9 @@ module Html2rss
         requested_url: url_obj,
         final_url: url_obj,
         status: nil,
-        verdict: Verdict.coerce(:drop),
+        verdict: Verdict[:drop],
         native_feed: nil,
-        surface_category: SurfaceCategory.coerce(:unsupported_surface),
+        surface_category: SurfaceCategory[:unsupported_surface],
         articles_count: 0,
         scheme_downgrade: false,
         notes: ["error: #{error.class} - #{error.message}"],

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -9,11 +9,11 @@ module Html2rss
     # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
     class BotasaurusContract
       # Closed set from OpenAPI ExecutionMode.
-      EXECUTION_MODES = %w[auto request browser].freeze
+      EXECUTION_MODES = Set['auto', 'request', 'browser'].freeze
       # Closed set from OpenAPI NavigationMode.
-      NAVIGATION_MODES = %w[auto get google_get google_get_bypass organic_get].freeze
+      NAVIGATION_MODES = Set['auto', 'get', 'google_get', 'google_get_bypass', 'organic_get'].freeze
       # Closed set from OpenAPI ErrorCategory.
-      ERROR_CATEGORIES = %w[timeout challenge_block navigation_error metadata_error validation].freeze
+      ERROR_CATEGORIES = Set['timeout', 'challenge_block', 'navigation_error', 'metadata_error', 'validation'].freeze
 
       # ScrapeRequest properties except +url+ (html2rss supplies the target URL).
       REQUEST_OPTION_KEYS = %i[
@@ -25,7 +25,7 @@ module Html2rss
       # OpenAPI WindowSize required keys (positive integers).
       WINDOW_SIZE_PROPERTIES = %i[width height].freeze
 
-      # Faraday POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
+      # HTTPX POST /scrape cap mirroring botasaurus-scrape-api total scrape wall (`SCRAPE_TIMEOUT_SECONDS`).
       SCRAPE_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_TIMEOUT_SECONDS', 45))
       # Post-boot navigate/wait budget mirrored from scrape-api (`SCRAPE_WORK_TIMEOUT_SECONDS`).
       SCRAPE_WORK_TIMEOUT_SECONDS = Integer(ENV.fetch('BOTASAURUS_SCRAPE_WORK_TIMEOUT_SECONDS', 30))
@@ -72,10 +72,7 @@ module Html2rss
 
         # @return [Hash{String => String}] normalized response headers (null → {})
         def headers
-          raw_headers = payload['headers']
-          return {} unless raw_headers.is_a?(Hash) && raw_headers.any?
-
-          raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
+          stringify_headers(payload['headers'])
         end
 
         # @return [Integer, nil] document status_code when present
@@ -147,10 +144,12 @@ module Html2rss
           entry[key.to_s] || entry[key]
         end
 
-        def xhr_headers(raw)
-          return {} unless raw.is_a?(Hash)
+        def xhr_headers(raw) = stringify_headers(raw)
+
+        def stringify_headers(raw)
+          return {} unless raw.is_a?(Hash) && raw.any?
 
-          raw.to_h { |key, value| [key.to_s, value.to_s] }
+          raw.transform_keys(&:to_s).transform_values(&:to_s)
         end
 
         def xhr_status_code(value)
@@ -286,7 +285,7 @@ module Html2rss
         payload
       end
 
-      # @param transport_response [Faraday::Response] upstream HTTP response
+      # @param transport_response [HTTPX::Response] upstream HTTP response
       # @return [Success, Error]
       # @raise [BotasaurusServiceError] when payload is not a scrape envelope
       def parse_response(transport_response)

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require 'faraday'
+require 'httpx'
 require 'json'
 require 'securerandom'
 
@@ -11,7 +11,7 @@ module Html2rss
     class BotasaurusStrategy < Strategy
       # Content-Type negotiation for the scrape API transport hop.
       TRANSPORT_ACCEPT = 'application/json'
-      # Disable compressed bodies so Faraday returns raw JSON without implicit decoding surprises.
+      # Disable compressed bodies so HTTPX returns raw JSON without implicit decoding surprises.
       TRANSPORT_ENCODING = 'identity'
       # Correlates each POST /scrape with botasaurus-scrape-api request logs.
       REQUEST_ID_HEADER = 'X-Request-Id'
@@ -28,10 +28,15 @@ module Html2rss
       def post_scrape_request
         request_id = SecureRandom.uuid
         Log.debug("#{self.class}: POST /scrape #{REQUEST_ID_HEADER}=#{request_id}")
-        transport_response = client.post('/scrape', JSON.generate(contract.request_payload), post_headers(request_id))
-        contract.parse_response(transport_response)
+        payload = JSON.generate(contract.request_payload)
+        response = client.post(scrape_endpoint, body: payload, headers: post_headers(request_id))
+        raise response.error if response.is_a?(HTTPX::ErrorResponse)
+
+        contract.parse_response(response)
       end
 
+      def scrape_endpoint = "#{scraper_base_url.to_s.chomp('/')}/scrape"
+
       def build_response(parsed_response)
         Response.new(
           body: parsed_response.html,
@@ -44,22 +49,14 @@ module Html2rss
       end
 
       def raise_from_error!(error)
-        raise_if_challenge_blocked!(error)
-        raise_if_timed_out!(error)
-        raise BotasaurusServiceError, error.failure_message
-      end
-
-      def raise_if_challenge_blocked!(error)
-        return unless error.challenge_block?
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}" if error.challenge_block?
 
-        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}"
-      end
-
-      def raise_if_timed_out!(error)
-        return unless error.timeout?
+        if error.timeout?
+          log_timeout!(reason: 'botasaurus_upstream')
+          raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
+        end
 
-        log_timeout!(reason: 'botasaurus_upstream')
-        raise RequestTimedOut.new(error.failure_message, timeout_phase: error.timeout_phase)
+        raise BotasaurusServiceError, error.failure_message
       end
 
       def response_url(final_url)
@@ -79,8 +76,18 @@ module Html2rss
       end
 
       def client
-        # No :gzip middleware on this client — compression is for remote target fetches only.
-        @client ||= Faraday.new(url: scraper_base_url.to_s, headers: client_headers, request: request_options)
+        timeout = attempt_timeout_seconds.to_f
+        @client ||= HTTPX.with(headers: client_headers, timeout: client_timeouts(timeout), resolver_class: :system)
+      end
+
+      def client_timeouts(timeout)
+        {
+          operation_timeout: timeout,
+          connect_timeout: [ctx.policy.connect_timeout_seconds, timeout].min,
+          read_timeout: timeout,
+          request_timeout: timeout,
+          total_request_timeout: timeout
+        }
       end
 
       def client_headers
@@ -92,14 +99,7 @@ module Html2rss
       end
 
       def post_headers(request_id)
-        {
-          'Content-Type' => 'application/json',
-          REQUEST_ID_HEADER => request_id
-        }
-      end
-
-      def request_options
-        { timeout: attempt_timeout_seconds.to_i }
+        { 'Content-Type' => 'application/json', REQUEST_ID_HEADER => request_id }
       end
 
       def attempt_timeout_seconds
@@ -123,6 +123,10 @@ module Html2rss
       def translate_connection_error(error)
         raise BotasaurusConnectionFailed, "Botasaurus connection failed: #{error.message}"
       end
+
+      def timeout_error?(error) = error.is_a?(HTTPX::TimeoutError) || super
+
+      def connection_error?(error) = error.is_a?(HTTPX::ConnectionError) || error.is_a?(HTTPX::TLSError) || super
     end
   end
 end

data/lib/html2rss/request_service/compressed_body.rb

@@ -45,9 +45,12 @@ module Html2rss
 
       def inflate(raw, encoding)
         case encoding
-        when 'gzip' then uncompress_gzip(raw)
-        when 'deflate' then inflate_deflate(raw)
+        when 'gzip'
+          return raw unless raw.start_with?(GZIP_MAGIC)
+
+          uncompress_gzip(raw)
         when 'br' then try_brotli(raw, require_html: false)
+        when 'deflate' then inflate_deflate(raw)
         end
       rescue Zlib::Error, Brotli::Error, ArgumentError
         nil
@@ -55,12 +58,6 @@ module Html2rss
       module_function :inflate
       private_class_method :inflate
 
-      def uncompress_gzip(raw)
-        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
-      end
-      module_function :uncompress_gzip
-      private_class_method :uncompress_gzip
-
       def inflate_deflate(raw)
         inflater = nil
         Zlib::Inflate.inflate(raw)
@@ -73,6 +70,14 @@ module Html2rss
       module_function :inflate_deflate
       private_class_method :inflate_deflate
 
+      def uncompress_gzip(raw)
+        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
+      rescue Zlib::Error
+        nil
+      end
+      module_function :uncompress_gzip
+      private_class_method :uncompress_gzip
+
       def try_brotli(raw, require_html: true)
         inflated = Brotli.inflate(raw)
         return inflated unless require_html
@@ -86,7 +91,10 @@ module Html2rss
       private_class_method :try_brotli
 
       def content_encodings(headers)
-        header(headers, 'content-encoding').to_s.split(',').map { |value| value.strip.downcase }.reject(&:empty?)
+        header(headers, 'content-encoding')
+          .to_s
+          .split(',')
+          .filter_map { |val| (s = val.strip.downcase).empty? ? nil : s }
       end
       module_function :content_encodings
       private_class_method :content_encodings