html2rss 0.26.0 → 0.27.0

data/lib/html2rss/request_service/botasaurus_contract.rb

@@ -5,43 +5,40 @@ require 'json'
 module Html2rss
   class RequestService
     ##
-    # Maps html2rss request/response handling to the botasaurus-scrape-api contract.
-    class BotasaurusContract
-      # Default Botasaurus scrape options when no explicit config is provided.
-      DEFAULT_OPTIONS = {
-        execution_mode: 'auto',
-        navigation_mode: 'auto',
-        max_retries: 1,
-        headless: false
-      }.freeze
-
-      # Allowlisted request.botasaurus keys forwarded to upstream.
-      OPTION_KEYS = %i[
-        execution_mode
-        navigation_mode
-        max_retries
-        wait_for_selector
-        wait_timeout_seconds
-        scroll
-        scroll_to_bottom
-        block_images
-        block_images_and_css
-        block_trackers
-        wait_for_complete_page_load
-        headless
-        proxy
-        user_agent
-        window_size
-        lang
-        headers
-        cookies
+    # Maps html2rss request/response handling to botasaurus-scrape-api OpenAPI 2.0
+    # +ScrapeRequest+ / +ScrapeSuccess+ / +ScrapeError+ (sibling +openapi.yaml+).
+    class BotasaurusContract # rubocop:disable Metrics/ClassLength -- Success/Error envelopes stay with the owner
+      # Closed set from OpenAPI ExecutionMode.
+      EXECUTION_MODES = %w[auto request browser].freeze
+      # Closed set from OpenAPI NavigationMode.
+      NAVIGATION_MODES = %w[auto get google_get google_get_bypass organic_get].freeze
+      # Closed set from OpenAPI ErrorCategory.
+      ERROR_CATEGORIES = %w[timeout challenge_block navigation_error metadata_error validation].freeze
+
+      # ScrapeRequest properties except +url+ (html2rss supplies the target URL).
+      REQUEST_OPTION_KEYS = %i[
+        execution_mode navigation_mode max_retries wait_for_selector wait_timeout_seconds
+        scroll block_images block_images_and_css block_trackers
+        wait_for_complete_page_load user_agent headers cookies window_size lang headless proxy
       ].freeze
 
-      # Allowlisted upstream response keys exposed as Response#transport_meta.
-      META_KEYS = %w[
-        request_id strategy_used render_ms challenge_detected blocked_detected attempts error_category
-        execution_tier detected_challenge
-      ].freeze
+      # OpenAPI WindowSize required keys (positive integers).
+      WINDOW_SIZE_PROPERTIES = %i[width height].freeze
+
+      # Published wait clamp floor (`[1, 20]` in OpenAPI description).
+      MIN_WAIT_TIMEOUT_SECONDS = 1
+      # Published wait clamp ceiling; YAML values above this are rejected.
+      MAX_WAIT_TIMEOUT_SECONDS = 20
+      # OpenAPI wait_timeout_seconds default (omitted from the client payload).
+      DEFAULT_WAIT_TIMEOUT_SECONDS = 15
+
+      # OpenAPI max_retries maximum (default 2 is applied upstream when omitted).
+      MAX_RETRIES = 3
+
+      # Allowlisted ScrapeDiagnostics keys nested under diagnostics.
+      DIAGNOSTICS_KEYS = %w[request_id attempts strategy_used render_ms execution_tier challenge].freeze
+      # Allowlisted ChallengeSignal keys nested under diagnostics.challenge.
+      CHALLENGE_KEYS = %w[blocked detected marker].freeze
 
       # Remaining seconds at or below which Botasaurus retries are disabled.
       TIGHT_BUDGET_SECONDS = 12
@@ -49,72 +46,52 @@ module Html2rss
       # Seconds reserved from remaining budget before setting wait_timeout_seconds.
       BUDGET_WAIT_RESERVE_SECONDS = 2
 
-      # Parsed Botasaurus response wrapper.
-      class ParsedResponse
-        # Fallback headers when upstream omits response headers.
-        DEFAULT_HEADERS = { 'content-type' => 'text/html' }.freeze
+      ##
+      # Parsed OpenAPI ScrapeSuccess envelope (HTTP 200).
+      class Success
         # Per-body size cap for captured XHR JSON (defense-in-depth vs upstream).
         MAX_XHR_BODY_BYTES = 500_000
         # Aggregate size cap across all captured XHR bodies.
         MAX_XHR_AGGREGATE_BYTES = 2_000_000
 
-        # @param payload [Hash{String => Object}] parsed Botasaurus response payload
-        # @param transport_status [Integer] HTTP status returned by Botasaurus
-        def initialize(payload:, transport_status:)
+        # @param payload [Hash{String => Object}] parsed ScrapeSuccess object
+        def initialize(payload:)
           @payload = payload
-          @transport_status = transport_status
-        end
-
-        # @return [Boolean] true when upstream classified request as challenge blocked
-        def challenge_block? = error_category == 'challenge_block'
-
-        # @return [Boolean] true when upstream returned non-200 or an error payload
-        def upstream_failure?
-          transport_status != 200 || status != 200 || error_message?
-        end
-
-        # @return [String] normalized challenge error message
-        def challenge_message
-          error || 'Botasaurus challenge block detected.'
-        end
-
-        # @return [String] actionable upstream failure summary
-        def upstream_failure_message
-          details = ["status=#{status}"]
-          details << "error_category=#{error_category}" if error_category
-          details << "error=#{error}" if error
-          details << "request_id=#{request_id}" if request_id
-          "Botasaurus scrape failed (#{details.join(', ')})."
+          html
+          diagnostics
         end
 
-        # @return [String] rendered HTML body from Botasaurus
-        # @raise [BotasaurusServiceError] when html is missing
+        # @return [String] rendered HTML body
         def html
           value = payload['html']
-          raise BotasaurusServiceError, "Botasaurus response missing required 'html' field" if value.nil?
+          raise BotasaurusServiceError, 'Botasaurus scrape success requires html' unless value.is_a?(String)
 
-          value.to_s
+          value
         end
 
-        # @return [Hash{String => String}] normalized response headers
+        # @return [Hash{String => String}] normalized response headers (null → {})
         def headers
           raw_headers = payload['headers']
-          return DEFAULT_HEADERS.dup unless raw_headers.is_a?(Hash) && raw_headers.any?
+          return {} unless raw_headers.is_a?(Hash) && raw_headers.any?
 
           raw_headers.to_h { |key, value| [key.to_s, value.to_s] }
         end
 
-        # @return [Integer] resolved status code (payload status_code or transport status)
+        # @return [Integer, nil] document status_code when present
         def status
           status_code = payload['status_code']
-          status_code.is_a?(Integer) ? status_code : transport_status
+          status_code.is_a?(Integer) ? status_code : nil
         end
 
         # @return [String, nil] final URL reported by upstream
         def final_url = payload['final_url']
 
-        # @return [Hash{String => Object}] allowlisted upstream telemetry (frozen)
-        def transport_meta = payload.slice(*META_KEYS).compact.freeze
+        # @return [Hash{String => Object}] diagnostics plus success-only metadata_error (frozen)
+        def transport_meta
+          meta = diagnostics.dup
+          meta['metadata_error'] = metadata_error if metadata_error
+          meta.freeze
+        end
 
         # @return [Array<Hash{String => Object}>] size-capped XHR/fetch captures (absent → [])
         def xhr_responses
@@ -133,17 +110,13 @@ module Html2rss
 
         private
 
-        attr_reader :payload, :transport_status
-
-        def error = payload['error']
+        attr_reader :payload
 
-        def request_id = payload['request_id']
+        def diagnostics = BotasaurusContract.diagnostics_from(payload)
 
-        def error_category = payload['error_category']
-
-        def error_message?
-          value = error
-          value.is_a?(String) ? !value.empty? : !value.nil?
+        def metadata_error
+          value = payload['metadata_error']
+          value.is_a?(String) && !value.empty? ? value : nil
         end
 
         def normalize_xhr_entry(entry)
@@ -184,6 +157,91 @@ module Html2rss
         end
       end
 
+      ##
+      # Parsed OpenAPI ScrapeError envelope (HTTP 400/403/422/502/504).
+      class Error
+        # @param payload [Hash{String => Object}] parsed ScrapeError object
+        # @param transport_status [Integer] HTTP status returned by Botasaurus
+        def initialize(payload:, transport_status:)
+          @payload = payload
+          @transport_status = transport_status
+          error
+          error_category
+          diagnostics
+        end
+
+        # @return [Boolean] true when upstream classified request as challenge blocked
+        def challenge_block? = error_category == 'challenge_block'
+
+        # @return [Boolean] true when the scrape envelope reports a timeout
+        def timeout?
+          transport_status == 504 || error_category == 'timeout'
+        end
+
+        # @return [String] normalized challenge error message
+        def challenge_message
+          [error, challenge_marker].find { |value| value.is_a?(String) && !value.empty? } ||
+            'Botasaurus challenge block detected.'
+        end
+
+        # @return [String] actionable upstream failure summary
+        def failure_message
+          details = ["status=#{transport_status}", "error_category=#{error_category}", "error=#{error}"]
+          details << "request_id=#{request_id}" if request_id
+          "Botasaurus scrape failed (#{details.join(', ')})."
+        end
+
+        private
+
+        attr_reader :payload, :transport_status
+
+        def error
+          value = payload['error']
+          raise BotasaurusServiceError, 'Botasaurus scrape error requires error' unless value.is_a?(String)
+
+          value
+        end
+
+        def error_category
+          value = payload['error_category']
+          return value if ERROR_CATEGORIES.include?(value)
+
+          raise BotasaurusServiceError, 'Botasaurus scrape error requires error_category'
+        end
+
+        def diagnostics = BotasaurusContract.diagnostics_from(payload)
+
+        def request_id = diagnostics['request_id']
+
+        def challenge_marker
+          challenge = diagnostics['challenge']
+          challenge.is_a?(Hash) ? challenge['marker'] : nil
+        end
+      end
+
+      ##
+      # @param payload [Hash{String => Object}] scrape envelope
+      # @return [Hash{String => Object}] allowlisted diagnostics (frozen)
+      # @raise [BotasaurusServiceError] when diagnostics or request_id are missing
+      def self.diagnostics_from(payload)
+        raw = payload['diagnostics']
+        raise BotasaurusServiceError, 'Botasaurus scrape envelope requires diagnostics' unless raw.is_a?(Hash)
+        raise BotasaurusServiceError, 'Botasaurus diagnostics require request_id' if raw['request_id'].to_s.empty?
+
+        sliced = raw.slice(*DIAGNOSTICS_KEYS)
+        sliced['challenge'] = compact_challenge(sliced['challenge'])
+        sliced.compact.freeze
+      end
+
+      # @param challenge [Object] diagnostics.challenge value
+      # @return [Hash{String => Object}, nil]
+      def self.compact_challenge(challenge)
+        return unless challenge.is_a?(Hash)
+
+        challenge.slice(*CHALLENGE_KEYS).compact
+      end
+      private_class_method :compact_challenge
+
       ##
       # @param url [Html2rss::Url] canonical URL to scrape
       # @param headers [Hash] request headers from context
@@ -194,6 +252,7 @@ module Html2rss
       # @option options [Integer] :max_retries
       # @option options [String] :wait_for_selector
       # @option options [Integer] :wait_timeout_seconds
+      # @option options [Boolean] :scroll
       # @option options [Boolean] :block_images
       # @option options [Boolean] :block_images_and_css
       # @option options [Boolean] :block_trackers
@@ -201,7 +260,7 @@ module Html2rss
       # @option options [Boolean] :headless
       # @option options [String] :proxy
       # @option options [String] :user_agent
-      # @option options [Array<Integer>] :window_size
+      # @option options [Hash] :window_size
       # @option options [String] :lang
       # @option options [Hash] :cookies
       # @option options [Hash] :headers
@@ -212,30 +271,37 @@ module Html2rss
         @remaining_timeout_seconds = remaining_timeout_seconds
       end
 
-      # @return [Hash] payload for POST /scrape (budget-clamped when remaining is known)
+      # @return [Hash] payload for POST /scrape (explicit options plus budget clamp-downs)
       def request_payload
-        payload = DEFAULT_OPTIONS.merge(filtered_options)
+        payload = { url: url.to_s }.merge(filtered_options)
         forwarded_headers = merged_headers
         payload[:headers] = forwarded_headers if forwarded_headers&.any?
-        payload.merge(url: url.to_s).then { clamp_for_budget(_1) }
+        clamp_for_budget(payload)
       end
 
       # @param transport_response [Faraday::Response] upstream HTTP response
-      # @return [ParsedResponse]
-      # @raise [BotasaurusServiceError] when payload is not valid JSON object
+      # @return [Success, Error]
+      # @raise [BotasaurusServiceError] when payload is not a scrape envelope
       def parse_response(transport_response)
-        payload = JSON.parse(transport_response.body.to_s)
-        raise BotasaurusServiceError, 'Botasaurus response must be a JSON object' unless payload.is_a?(Hash)
+        payload = json_object(transport_response.body.to_s)
+        return Success.new(payload:) if transport_response.status == 200
 
-        ParsedResponse.new(payload:, transport_status: transport_response.status)
-      rescue JSON::ParserError => error
-        raise BotasaurusServiceError, "Botasaurus response JSON parse failed: #{error.message}"
+        Error.new(payload:, transport_status: transport_response.status)
       end
 
       private
 
       attr_reader :url, :headers, :options, :remaining_timeout_seconds
 
+      def json_object(body)
+        payload = JSON.parse(body)
+        return payload if payload.is_a?(Hash)
+
+        raise BotasaurusServiceError, 'Botasaurus response must be a JSON object'
+      rescue JSON::ParserError => error
+        raise BotasaurusServiceError, "Botasaurus response JSON parse failed: #{error.message}"
+      end
+
       def merged_headers
         explicit = options[:headers] || {}
         base = headers.is_a?(Hash) ? headers : {}
@@ -244,22 +310,47 @@ module Html2rss
       end
 
       def filtered_options
-        OPTION_KEYS.each_with_object({}) do |key, normalized|
+        REQUEST_OPTION_KEYS.each_with_object({}) do |key, normalized|
           normalized[key] = options[key] if options.key?(key)
         end
       end
 
       def clamp_for_budget(payload)
         remaining = remaining_timeout_seconds
-        return payload if remaining.nil?
-
         clamped = payload.dup
-        clamped[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
-        budget_wait = [1, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
-        configured = clamped[:wait_timeout_seconds]
-        clamped[:wait_timeout_seconds] = configured ? [configured, budget_wait].min : budget_wait
+        unless remaining.nil?
+          apply_tight_retries!(clamped, remaining)
+          apply_budget_wait!(clamped, remaining)
+        end
+        cap_wait_timeout!(clamped)
         clamped
       end
+
+      def apply_tight_retries!(payload, remaining)
+        payload[:max_retries] = 0 if remaining <= TIGHT_BUDGET_SECONDS
+      end
+
+      def apply_budget_wait!(payload, remaining)
+        budget_wait = budget_wait_seconds(remaining)
+        configured = payload[:wait_timeout_seconds]
+        if configured
+          payload[:wait_timeout_seconds] = [configured, budget_wait].min
+        elsif budget_wait < DEFAULT_WAIT_TIMEOUT_SECONDS
+          payload[:wait_timeout_seconds] = budget_wait
+        end
+      end
+
+      def budget_wait_seconds(remaining)
+        reserved = [MIN_WAIT_TIMEOUT_SECONDS, (remaining - BUDGET_WAIT_RESERVE_SECONDS).floor].max
+        [reserved, MAX_WAIT_TIMEOUT_SECONDS].min
+      end
+
+      def cap_wait_timeout!(payload)
+        wait = payload[:wait_timeout_seconds]
+        return unless wait
+
+        payload[:wait_timeout_seconds] = [wait, MAX_WAIT_TIMEOUT_SECONDS].min
+      end
     end
   end
 end

data/lib/html2rss/request_service/botasaurus_strategy.rb

@@ -12,8 +12,8 @@ module Html2rss
 
       def fetch
         parsed_response = post_scrape_request
-        raise_if_challenge_blocked!(parsed_response)
-        raise_if_upstream_failed!(parsed_response)
+        raise_from_error!(parsed_response) if parsed_response.is_a?(BotasaurusContract::Error)
+
         build_response(parsed_response)
       end
 
@@ -33,16 +33,23 @@ module Html2rss
         )
       end
 
-      def raise_if_challenge_blocked!(parsed_response)
-        return unless parsed_response.challenge_block?
+      def raise_from_error!(error)
+        raise_if_challenge_blocked!(error)
+        raise_if_timed_out!(error)
+        raise BotasaurusServiceError, error.failure_message
+      end
+
+      def raise_if_challenge_blocked!(error)
+        return unless error.challenge_block?
 
-        raise BlockedSurfaceDetected, "Blocked surface detected: #{parsed_response.challenge_message}"
+        raise BlockedSurfaceDetected, "Blocked surface detected: #{error.challenge_message}"
       end
 
-      def raise_if_upstream_failed!(parsed_response)
-        return unless parsed_response.upstream_failure?
+      def raise_if_timed_out!(error)
+        return unless error.timeout?
 
-        raise BotasaurusServiceError, parsed_response.upstream_failure_message
+        log_timeout!(reason: 'botasaurus_upstream')
+        raise RequestTimedOut, error.failure_message
       end
 
       def response_url(final_url)

data/lib/html2rss/request_service/compressed_body.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+require 'brotli'
+require 'stringio'
+require 'zlib'
+
+module Html2rss
+  class RequestService
+    # Decodes gzip/brotli bodies when Content-Encoding is missing or the type is octet-stream.
+    module CompressedBody
+      # Gzip stream magic (RFC 1952).
+      GZIP_MAGIC = "\x1F\x8B".b.freeze
+      # HTML prefix for unlabeled brotli trial decode. Same object as {Response::HTML_BODY_SNIFF}.
+      HTML_BODY_SNIFF = Response::HTML_BODY_SNIFF
+
+      module_function
+
+      # @param body [String, nil] raw response body
+      # @param headers [Hash] response headers (Content-Encoding / Content-Type)
+      # @return [String, nil] decoded body when compressed; otherwise the original body
+      def decode(body, headers: {})
+        return body if body.nil? || body.empty?
+
+        raw = body.to_s.b
+        decode_labeled(raw, headers) || decode_unlabeled(raw, headers) || body
+      end
+
+      def decode_labeled(raw, headers)
+        encodings = content_encodings(headers)
+        return if encodings.empty?
+
+        encodings.reverse.reduce(raw) { |buffer, encoding| inflate(buffer, encoding) || buffer }
+      end
+      module_function :decode_labeled
+      private_class_method :decode_labeled
+
+      def decode_unlabeled(raw, headers)
+        return uncompress_gzip(raw) if raw.start_with?(GZIP_MAGIC)
+        return unless octet_stream?(headers)
+
+        try_brotli(raw)
+      end
+      module_function :decode_unlabeled
+      private_class_method :decode_unlabeled
+
+      def inflate(raw, encoding)
+        case encoding
+        when 'gzip' then uncompress_gzip(raw)
+        when 'deflate' then inflate_deflate(raw)
+        when 'br' then try_brotli(raw, require_html: false)
+        end
+      rescue Zlib::Error, Brotli::Error, ArgumentError
+        nil
+      end
+      module_function :inflate
+      private_class_method :inflate
+
+      def uncompress_gzip(raw)
+        Zlib::GzipReader.wrap(StringIO.new(raw), encoding: 'ASCII-8BIT', &:read)
+      end
+      module_function :uncompress_gzip
+      private_class_method :uncompress_gzip
+
+      def inflate_deflate(raw)
+        inflater = nil
+        Zlib::Inflate.inflate(raw)
+      rescue Zlib::DataError
+        inflater = Zlib::Inflate.new(-Zlib::MAX_WBITS)
+        inflater.inflate(raw)
+      ensure
+        inflater&.close
+      end
+      module_function :inflate_deflate
+      private_class_method :inflate_deflate
+
+      def try_brotli(raw, require_html: true)
+        inflated = Brotli.inflate(raw)
+        return inflated unless require_html
+        return inflated if inflated.b.match?(HTML_BODY_SNIFF)
+
+        nil
+      rescue Brotli::Error, ArgumentError
+        nil
+      end
+      module_function :try_brotli
+      private_class_method :try_brotli
+
+      def content_encodings(headers)
+        header(headers, 'content-encoding').to_s.split(',').map { |value| value.strip.downcase }.reject(&:empty?)
+      end
+      module_function :content_encodings
+      private_class_method :content_encodings
+
+      def octet_stream?(headers)
+        header(headers, 'content-type').to_s.downcase.include?('application/octet-stream')
+      end
+      module_function :octet_stream?
+      private_class_method :octet_stream?
+
+      def header(headers, name)
+        headers.fetch(name) do
+          headers.find { |key, _value| key.to_s.casecmp?(name) }&.last
+        end
+      end
+      module_function :header
+      private_class_method :header
+    end
+  end
+end

data/lib/html2rss/request_service/faraday_strategy.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require 'brotli'
 require 'faraday'
 require 'faraday/follow_redirects'
 require 'faraday/gzip'
@@ -89,7 +90,8 @@ module Html2rss
       end
 
       def build_response(response)
-        Response.new(body: response.body, headers: response.headers, url: response_url(response),
+        Response.new(body: CompressedBody.decode(response.body, headers: response.headers),
+                     headers: response.headers, url: response_url(response),
                      status: response.status)
       end
 

data/lib/html2rss/request_service/policy.rb

@@ -115,7 +115,7 @@ module Html2rss
       # @raise [UnsupportedUrlScheme] if the redirect downgrades from HTTPS to HTTP
       def validate_redirect!(from_url:, to_url:, origin_url:, relation:)
         if from_url.scheme == 'https' && to_url.scheme == 'http'
-          raise UnsupportedUrlScheme, 'Redirect downgraded from https to http'
+          raise UnsupportedUrlScheme, "Redirect downgraded from #{from_url} to #{to_url}"
         end
 
         validate_request!(url: to_url, origin_url:, relation:)

data/lib/html2rss/request_service/response.rb

@@ -12,6 +12,14 @@ module Html2rss
       # Default when a strategy does not capture sub-resource responses.
       EMPTY_CAPTURED_RESPONSES = [].freeze
 
+      # Bodies that look like HTML even when Content-Type is missing, empty, or wrong.
+      HTML_BODY_SNIFF = /\A\s*(?:<!DOCTYPE\s+html|<html)/i
+      # Charset from Content-Type or a leading <meta charset>.
+      CHARSET_PARAMETER = /charset\s*=\s*["']?([\w.:-]+)/i
+      # Bytes scanned for a meta charset hint (HTML spec looks at the first 1024).
+      META_CHARSET_BYTES = 2048
+      private_constant :CHARSET_PARAMETER, :META_CHARSET_BYTES
+
       ##
       # @param body [String] the body of the response
       # @param url [Html2rss::Url] the final request URL
@@ -64,18 +72,17 @@ module Html2rss
       # @return [Boolean] whether response content is JSON
       def json_response? = content_type.include?('application/json')
 
-      # @return [Boolean] whether response content is HTML
-      def html_response? = content_type.include?('text/html')
+      # @return [Boolean] whether response content is HTML (header or sniffed body, never JSON)
+      def html_response?
+        content_type.include?('text/html') || (!json_response? && html_looking_body?)
+      end
 
       ##
       # @return [Nokogiri::HTML::Document, Hash] the parsed body of the response, frozen object
       # @raise [UnsupportedResponseContentType] if the content type is not supported
       def parsed_body
         @parsed_body ||= if html_response?
-                           Nokogiri::HTML(body).tap do |doc|
-                             # Remove comments from the document to avoid processing irrelevant content
-                             doc.xpath('//comment()').each(&:remove)
-                           end.freeze
+                           parse_html_document
                          elsif json_response?
                            JSON.parse(body, symbolize_names: true).freeze
                          else
@@ -92,6 +99,50 @@ module Html2rss
           headers.find { |key, _value| key.casecmp?(name) }&.last
         end
       end
+
+      def parse_html_document
+        Nokogiri::HTML(decoded_html_body).tap do |doc|
+          doc.xpath('//comment()').each(&:remove)
+        end.freeze
+      end
+
+      def decoded_html_body
+        bytes = body.to_s.b
+        transcode_html(bytes, header_charset || meta_charset(bytes))
+      end
+
+      def transcode_html(bytes, charset)
+        encoding = html_encoding_for(charset)
+        return utf8_scrub(bytes) unless encoding
+
+        bytes.dup.force_encoding(encoding).encode(Encoding::UTF_8)
+      rescue Encoding::UndefinedConversionError, Encoding::InvalidByteSequenceError
+        utf8_scrub(bytes)
+      end
+
+      def html_encoding_for(charset)
+        return if charset.nil? || utf8_charset?(charset)
+
+        Encoding.find(charset)
+      rescue ArgumentError
+        nil
+      end
+
+      def utf8_charset?(charset) = charset.to_s.downcase.gsub(/[\s_-]/, '') == 'utf8'
+
+      def utf8_scrub(bytes) = bytes.dup.force_encoding(Encoding::UTF_8).scrub
+
+      def header_charset
+        content_type[CHARSET_PARAMETER, 1]
+      end
+
+      def meta_charset(bytes)
+        bytes.byteslice(0, META_CHARSET_BYTES).to_s[CHARSET_PARAMETER, 1]
+      end
+
+      def html_looking_body?
+        body.to_s.b.match?(HTML_BODY_SNIFF)
+      end
     end
   end
 end