fetch_util 0.4.0 → 0.5.1

data/lib/fetch_util/browser/stabilization/anubis.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module FetchUtil
+  class Browser
+    module Stabilization
+      module Anubis
+        ANUBIS_POLL = 0.25
+
+        private
+
+        def wait_for_anubis_challenge(page)
+          deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @timeout
+          state = anubis_page_state(page)
+          if !valid_anubis_state?(state) && Process.clock_gettime(Process::CLOCK_MONOTONIC) < deadline
+            state = anubis_page_state(page)
+          end
+          return false unless valid_anubis_state?(state) && state["challenge"] == true
+
+          remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          return false unless remaining.positive?
+
+          retry_until_timeout(remaining, interval: ANUBIS_POLL) do
+            state = anubis_page_state(page)
+            next false unless valid_anubis_state?(state)
+            next false unless state["challenge"] == false
+
+            state["document_ready"] == true && state["body_present"] == true && state["body_text_present"] == true
+          end
+        end
+
+        def valid_anubis_state?(state)
+          state.is_a?(Hash) &&
+            [true, false].include?(state["challenge"]) &&
+            [true, false].include?(state["document_ready"]) &&
+            [true, false].include?(state["body_present"]) &&
+            [true, false].include?(state["body_text_present"]) &&
+            state["url"].is_a?(String) && !state["url"].empty?
+        end
+
+        def anubis_page_state(page)
+          safe_evaluate(page, <<~JS, default: {})
+            (() => {
+              const body = document.body;
+              const bodyText = body ? (body.innerText || '').replace(/\s+/g, ' ').trim() : '';
+              const anubisRoot = document.querySelector('#anubis_challenge');
+              const anubisScript = document.querySelector('script[src*="/.within.website/x/cmd/anubis/"]');
+              const visibleRoot = anubisRoot && (() => {
+                const style = window.getComputedStyle(anubisRoot);
+                const rect = anubisRoot.getBoundingClientRect();
+                return style.display !== 'none' && style.visibility !== 'hidden' && rect.width > 0 && rect.height > 0;
+              })();
+              const makingSure = /making sure you're not a bot/i.test((document.title || '') + ' ' + bodyText);
+              const protectedChallenge = /protected by anubis/i.test(bodyText) &&
+                /(enable javascript to get past this challenge|please wait a moment while we ensure the security of your connection|loading|calculating|challenge:\s*anubis)/i.test(bodyText);
+              const challenge = Boolean((visibleRoot || anubisScript) && (makingSure || protectedChallenge));
+              return {
+                challenge,
+                document_ready: document.readyState === 'complete',
+                body_present: Boolean(body),
+                body_text_present: bodyText.length > 0,
+                url: location.href
+              };
+            })()
+          JS
+        end
+      end
+    end
+  end
+end

data/lib/fetch_util/browser/stabilization/page_flow.rb

@@ -38,6 +38,8 @@ module FetchUtil
         private
 
         def stabilize_page(page, url)
+          wait_for_anubis_challenge(page)
+
           if (profile = matching_stabilization_profile(url, PAGE_FLOW_STABILIZATION_PROFILES))
             return send(profile.fetch(:strategy), page)
           end

data/lib/fetch_util/browser/stabilization.rb

@@ -5,9 +5,11 @@ module FetchUtil
     module Stabilization
       autoload :PageFlow, "fetch_util/browser/stabilization/page_flow"
       autoload :SpaHydration, "fetch_util/browser/stabilization/spa_hydration"
+      autoload :Anubis, "fetch_util/browser/stabilization/anubis"
 
       include PageFlow
       include SpaHydration
+      include Anubis
     end
   end
 end

data/lib/fetch_util/cli.rb

@@ -71,7 +71,7 @@ module FetchUtil
     end
 
     desc "search QUERY", "Search across configured engines and aggregate results"
-    option :source, type: :array, default: FetchUtil::Searcher::DEFAULT_SOURCES, desc: "Search sources"
+    option :source, type: :array, default: nil, desc: "Search sources"
     option :limit, type: :numeric, default: nil, desc: "Maximum results; omit to return every result in the fetched response"
     option :verbose_search, type: :boolean, default: false, desc: "Include per-result search provenance"
     def search(*terms)
@@ -82,9 +82,8 @@ module FetchUtil
         request_log: request_log,
         sources: options[:source],
         limit: options[:limit],
-        concurrency: [options[:concurrency], options[:source].length].min,
         verbose: options[:verbose_search],
-        **fetch_options
+        timeout: options[:timeout]
       ).search(query)
 
       emit(payload)

data/lib/fetch_util/search_transport.rb

@@ -0,0 +1,579 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "nokogiri"
+require "stringio"
+require "timeout"
+require "uri"
+require "zlib"
+require "base64"
+
+module FetchUtil
+  # Fetches and parses supported search result pages without involving the browser fetcher.
+  class SearchTransport
+    Candidate = Data.define(:source, :title, :url, :snippet, :source_rank) do
+      def initialize(source:, title:, url:, source_rank:, snippet: nil)
+        super(source: source.to_s.freeze, title: title.to_s.freeze, url: url.to_s.freeze,
+              snippet: snippet&.to_s&.freeze, source_rank: Integer(source_rank))
+      end
+    end
+
+    SourceResponse = Data.define(:source, :transport, :status, :candidates, :elapsed_ms, :final_url, :reason) do
+      def initialize(source:, status:, elapsed_ms:, transport: "http", candidates: [], final_url: nil, reason: nil)
+        super(source: source.to_s.freeze, transport: transport.to_s.freeze, status: status.to_s.freeze,
+              candidates: candidates.freeze, elapsed_ms: Integer(elapsed_ms), final_url: final_url&.to_s&.freeze,
+              reason: reason&.to_s&.freeze)
+      end
+    end
+
+    HttpResponse = Data.define(:status, :headers, :body, :final_url)
+    HttpFailure = Data.define(:reason, :final_url)
+
+    SOURCES = {
+      "brave" => { url: "https://search.brave.com/search?q=%{query}", hosts: %w[search.brave.com] },
+      "bing" => { url: "https://www.bing.com/search?q=%{query}&setlang=en-US&cc=US", hosts: %w[www.bing.com cn.bing.com] },
+      "duckduckgo" => { url: "https://html.duckduckgo.com/html/?q=%{query}", hosts: %w[html.duckduckgo.com] },
+      "google" => { url: "https://www.google.com/search?q=%{query}", hosts: %w[www.google.com] },
+      "ecosia" => { url: "https://www.ecosia.org/search?q=%{query}", hosts: %w[www.ecosia.org] },
+      "yahoo" => { url: "https://search.yahoo.com/search?p=%{query}", hosts: %w[search.yahoo.com] }
+    }.freeze
+    DEFAULT_TIMEOUT = 10.0
+    YAHOO_RECOVERY_ATTEMPTS = 2
+    FAILURE_REASONS = %w[challenge failed host http_status parse query_mismatch redirect size timeout].freeze
+    RELEVANCE_HEALTH_CANDIDATE_COUNT = 3
+    QUERY_FUNCTION_WORDS = %w[
+      a an and are at be been being but by can could did do does for from had has have how i if in is it me my no not of on or our shall
+      should that the their them then there these they this to was we were what when where which who why will with would you your
+    ].freeze
+    SCOPED_QUERY_TERM = /(?:\A|\s)[+-]?[[:alpha:]][[:alnum:]_-]*:(?:"[^"]*"|'[^']*'|\S+)/
+    NEGATED_QUERY_TERM = /(?:\A|\s)-\S+/
+    WRAPPER_HOSTS = {
+      "bing" => %w[bing.com www.bing.com cn.bing.com],
+      "duckduckgo" => %w[duckduckgo.com www.duckduckgo.com html.duckduckgo.com],
+      "google" => %w[google.com www.google.com],
+      "yahoo" => %w[r.search.yahoo.com]
+    }.freeze
+
+    def initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil)
+      @sources = sources.map(&:to_s).freeze
+      unknown = @sources - SOURCES.keys
+      raise ArgumentError, "unknown search sources: #{unknown.join(", ")}" if unknown.any?
+
+      @timeout = Float(timeout)
+      raise ArgumentError, "timeout must be positive" unless @timeout.positive?
+
+      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+      @http_client = http_client || HttpClient.new(clock: @clock)
+      @html_parser = html_parser || ->(body) { Nokogiri::HTML(body) }
+    end
+
+    def self.candidates_match_query?(query, candidates)
+      candidates.first(RELEVANCE_HEALTH_CANDIDATE_COUNT).any? do |candidate|
+        candidate_matches_query?(query, candidate)
+      end
+    end
+
+    def self.candidate_matches_query?(query, candidate)
+      query_terms = meaningful_query_terms(normalized_query_text(query))
+      return true if query_terms.length < 3
+
+      candidate_terms = normalized_terms("#{candidate.title} #{candidate.snippet}")
+      (query_terms & candidate_terms).length >= 2
+    end
+
+    def search(query, timeout: @timeout)
+      query = query.to_s.strip
+      raise ArgumentError, "query must not be empty" if query.empty?
+
+      request_timeout = Float(timeout)
+      raise ArgumentError, "timeout must be positive" unless request_timeout.positive?
+
+      deadline = clock.call + request_timeout
+      responses = Array.new(sources.length)
+      threads = sources.each_with_index.map do |source, index|
+        Thread.new { responses[index] = search_source(source, query, deadline) }
+      end
+      threads.each(&:join)
+      responses
+    end
+
+    private
+
+    attr_reader :clock, :html_parser, :http_client, :sources, :timeout
+
+    def search_source(source, query, deadline)
+      started_at = clock.call
+      url = build_url(source, query)
+      result = yahoo_response(source, url, deadline)
+      if result.is_a?(HttpFailure)
+        return failure(source, result.reason, elapsed_since(started_at), result.final_url)
+      end
+      outcome = within_deadline(deadline) do
+        document = html_parser.call(result.body)
+        next [:failed, "challenge"] if challenge?(source, result, document)
+        next [:failed, "http_status"] unless result.status.between?(200, 299)
+        next [:empty] if no_results?(source, document)
+
+        candidates = parse_candidates(source, document)
+        next [:ok, candidates] if candidates.empty?
+        next [:suspect, candidates] unless self.class.candidates_match_query?(query, candidates)
+
+        [:ok, candidates]
+      end
+      elapsed_ms = elapsed_since(started_at)
+      return failure(source, outcome[1], elapsed_ms, result.final_url, candidates: outcome[2] || []) if outcome.first == :failed
+      return empty(source, elapsed_ms, result.final_url) if outcome.first == :empty
+      if outcome.first == :suspect
+        return SourceResponse.new(
+          source: source,
+          status: "ok",
+          candidates: outcome.last,
+          elapsed_ms: elapsed_ms,
+          final_url: result.final_url,
+          reason: "query_mismatch"
+        )
+      end
+
+      candidates = outcome.last
+      return failure(source, "parse", elapsed_ms, result.final_url) if candidates.empty?
+
+      SourceResponse.new(source: source, status: "ok", candidates: candidates, elapsed_ms: elapsed_ms, final_url: result.final_url)
+    rescue HttpClient::DeadlineExceeded
+      failure(source, "timeout", elapsed_since(started_at), url)
+    rescue Nokogiri::XML::SyntaxError
+      failure(source, "parse", elapsed_since(started_at), url)
+    rescue StandardError
+      # The public boundary intentionally maps implementation failures to a finite diagnostic code.
+      failure(source, "failed", elapsed_since(started_at), url)
+    end
+
+    def yahoo_response(source, url, deadline)
+      return http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts)) unless source == "yahoo"
+
+      result = http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts))
+      YAHOO_RECOVERY_ATTEMPTS.times do
+        break unless yahoo_retryable?(result) && deadline > clock.call
+
+        result = http_client.get(url, deadline: deadline, allowed_hosts: SOURCES.fetch(source).fetch(:hosts))
+      end
+      result
+    end
+
+    def yahoo_retryable?(result)
+      return result.reason == "failed" if result.is_a?(HttpFailure)
+
+      result.is_a?(HttpResponse) && (result.status == 429 || result.status.between?(500, 599))
+    end
+
+    def build_url(source, query)
+      SOURCES.fetch(source).fetch(:url) % { query: URI.encode_www_form_component(query) }
+    end
+
+    def parse_candidates(source, document)
+      result_nodes(source, document).filter_map do |node|
+        next if excluded_node?(source, node)
+
+        anchor = result_anchor(source, node)
+        next unless anchor
+
+        url = destination(source, anchor["href"])
+        title = normalized_text(title_node(source, node, anchor))
+        next if url.nil? || title.empty? || engine_url?(source, url)
+
+        snippet = normalized_text(node.at_css(snippet_selector(source)))
+        [title, url, snippet]
+      end.each_with_index.map do |(title, url, snippet), index|
+        Candidate.new(source: source, title: title, url: url, snippet: snippet.empty? ? nil : snippet, source_rank: index + 1)
+      end
+    end
+
+    def self.meaningful_query_terms(query)
+      semantic_query = query.gsub(SCOPED_QUERY_TERM, " ").gsub(NEGATED_QUERY_TERM, " ")
+      normalized_terms(semantic_query).reject { |term| QUERY_FUNCTION_WORDS.include?(term) }
+    end
+
+    def self.normalized_terms(text)
+      normalized = normalized_query_text(text)
+      normalized = normalized.gsub(/([[:upper:]]+)([[:upper:]][[:lower:]])/, '\1 \2')
+      normalized = normalized.gsub(/([[:lower:]\d])([[:upper:]])/, '\1 \2')
+      normalized.downcase.scan(/[[:alnum:]]+/).uniq
+    end
+
+    def self.normalized_query_text(text)
+      text.to_s.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ").unicode_normalize(:nfc)
+    end
+
+    private_class_method :meaningful_query_terms, :normalized_terms, :normalized_query_text
+
+    def result_nodes(source, document)
+      return brave_result_nodes(document) if source == "brave"
+      return yahoo_result_nodes(document) if source == "yahoo"
+
+      selector = {
+        "bing" => "#b_results li.b_algo",
+        "duckduckgo" => ".results .result, .result.results_links",
+        "google" => "#search .g, #search .MjjYud, #search [data-snhf]",
+        "ecosia" => ".result, article.result"
+      }.fetch(source)
+      nodes = document.css(selector).uniq
+      nodes.reject { |node| nodes.any? { |other| other != node && other.ancestors.include?(node) } }
+    end
+
+    def brave_result_nodes(document)
+      current = document.css(".snippet[data-type='web']")
+      legacy = document.css("#results > .snippet:not([id]):not([data-type])").select { |node| legacy_brave_result?(node) }
+      (current.to_a + legacy).uniq
+    end
+
+    def yahoo_result_nodes(document)
+      nodes = document.css("#web .algo, #web .algo-sr, #web .dd.algo").uniq
+      nodes.reject { |node| nodes.any? { |other| other != node && other.ancestors.include?(node) } }
+    end
+
+    def legacy_brave_result?(node)
+      node.at_css("h2, h3, .title.search-snippet-title") && result_anchor("brave", node)
+    end
+
+    def result_anchor(source, node)
+      case source
+      when "brave"
+        node.css("a[href]").find do |candidate|
+          url = destination(source, candidate["href"])
+          url && !engine_url?(source, url)
+        end
+      when "bing" then node.at_css("h2 a[href]")
+      when "duckduckgo" then node.at_css("a.result__a[href], h2 a[href]")
+      when "google" then node.at_css("a[href]:has(h3)") || node.at_css("h3")&.ancestors("a[href]")&.first
+      when "yahoo" then node.at_css("h3 a[href], .compTitle a[href]")
+      else node.at_css("h2 a[href], h3 a[href], a[href]:has(h3)")
+      end
+    end
+
+    def title_node(source, node, anchor)
+      return node.at_css(".title.search-snippet-title") || anchor.at_css("h2, h3") || anchor if source == "brave"
+
+      anchor.at_css("h2, h3") || anchor
+    end
+
+    def snippet_selector(source)
+      {
+        "brave" => ".content, .snippet-description, .snippet-content, .description",
+        "bing" => ".b_caption p, .b_paractl",
+        "duckduckgo" => ".result__snippet",
+        "google" => ".VwiC3b, .aCOpRe, [data-sncf]",
+        "ecosia" => ".result-snippet, .result__description",
+        "yahoo" => ".compText, .compText p"
+      }.fetch(source)
+    end
+
+    def excluded_node?(source, node)
+      return yahoo_excluded_node?(node) if source == "yahoo"
+
+      node.xpath("ancestor-or-self::*").any? do |ancestor|
+        value = [ancestor["class"], ancestor["id"], ancestor["data-testid"]].compact.join(" ").downcase
+        value.match?(/\b(ad|ads|advert|enrichment|knowledge|llm|nav|pagination|related|answer)\b/)
+      end
+    end
+
+    def yahoo_excluded_node?(node)
+      node.xpath("ancestor-or-self::*").any? do |ancestor|
+        value = [ancestor["class"], ancestor["id"], ancestor["data-testid"]].compact.join(" ").downcase
+        value.match?(/\b(ad|ads|advert|control|controls|enrichment|knowledge|llm|lookalike|nav|navigation|pagination|related|right[-_ ]?rail|assist)\b/)
+      end
+    end
+
+    def destination(source, href)
+      value = href.to_s.strip
+      value = decode_bing(value) if source == "bing"
+      value = decode_wrapper(source, value) if %w[google duckduckgo yahoo].include?(source)
+      uri = URI.parse(value)
+      return unless uri.is_a?(URI::HTTP) && uri.host
+
+      uri.to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def decode_bing(value)
+      uri = URI.parse(value)
+      return value unless uri.path == "/ck/a" && (uri.host.nil? || wrapper_host?("bing", uri.host))
+
+      encoded = URI.decode_www_form(uri.query.to_s).assoc("u")&.last.to_s
+      return value unless encoded.start_with?("a1")
+
+      decoded = encoded.delete_prefix("a1").tr("-_,", "+/=")
+      decoded += "=" * ((4 - decoded.length % 4) % 4)
+      Base64.strict_decode64(decoded).force_encoding("UTF-8").scrub
+    rescue ArgumentError, URI::InvalidURIError
+      value
+    end
+
+    def decode_wrapper(source, value)
+      uri = URI.parse(value)
+      return decode_yahoo_wrapper(uri, value) if source == "yahoo"
+
+      key = source == "google" ? %w[q url] : %w[uddg]
+      path = wrapper_path(source)
+      return value unless uri.path == path && (uri.host.nil? || wrapper_host?(source, uri.host))
+
+      URI.decode_www_form(uri.query.to_s).to_h.values_at(*key).compact.first || value
+    rescue URI::InvalidURIError
+      value
+    end
+
+    def decode_yahoo_wrapper(uri, value)
+      return value unless wrapper_host?("yahoo", uri.host)
+
+      segment = uri.path.split("/").find { |part| part.start_with?("RU=") }
+      return value unless segment
+
+      decoded = URI::DEFAULT_PARSER.unescape(segment.delete_prefix("RU="))
+      destination = URI.parse(decoded)
+      return nil unless destination.is_a?(URI::HTTP) && destination.host
+
+      destination.to_s
+    rescue URI::InvalidURIError
+      nil
+    end
+
+    def engine_url?(source, value)
+      host = URI.parse(value).host
+      same_engine_host?(source, host) || wrapper_host?(source, host)
+    rescue URI::InvalidURIError
+      true
+    end
+
+    def same_engine_host?(source, host)
+      SOURCES.fetch(source).fetch(:hosts).include?(host.to_s.downcase)
+    end
+
+    def wrapper_host?(source, host)
+      WRAPPER_HOSTS.fetch(source, []).include?(host.to_s.downcase)
+    end
+
+    def wrapper_path(source)
+      { "bing" => "/ck/a", "google" => "/url", "duckduckgo" => "/l/" }.fetch(source)
+    end
+
+    def normalized_text(node)
+      node ? node.text.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ").gsub(/\s+/, " ").strip : ""
+    end
+
+    def challenge?(source, response, document)
+      return true if source == "google" && URI.parse(response.final_url).path.start_with?("/sorry")
+      return true if source == "duckduckgo" && response.status == 202
+
+      title = normalized_text(document.at_css("title")).downcase
+      visible_text = visible_document_text(document)
+      "#{title} #{visible_text}".match?(/captcha|unusual traffic|verify you are human|consent/)
+    rescue URI::InvalidURIError
+      true
+    end
+
+    def no_results?(source, document)
+      text = visible_document_text(document)
+      patterns = {
+        "brave" => /no results|did not match any documents/,
+        "bing" => /there are no results|no results found/,
+        "duckduckgo" => /no results|no more results/,
+        "google" => /did not match any documents|no results found/,
+        "ecosia" => /no results found|we couldn't find/,
+        "yahoo" => /no results found|we couldn't find|did not match any results/
+      }
+      text.match?(patterns.fetch(source))
+    end
+
+    def visible_document_text(document)
+      visible = document.dup
+      visible.css("script, style, template, noscript").remove
+      normalized_text(visible).downcase
+    end
+
+    def failure(source, reason, elapsed_ms, final_url, candidates: [])
+      reason = reason.to_s
+      reason = "failed" unless FAILURE_REASONS.include?(reason)
+      SourceResponse.new(
+        source: source,
+        status: "failed",
+        candidates: candidates,
+        elapsed_ms: elapsed_ms,
+        final_url: final_url,
+        reason: reason
+      )
+    end
+
+    def empty(source, elapsed_ms, final_url)
+      SourceResponse.new(source: source, status: "empty", elapsed_ms: elapsed_ms, final_url: final_url)
+    end
+
+    def elapsed_since(started_at)
+      ((clock.call - started_at) * 1000).round
+    end
+
+    def within_deadline(deadline, &block)
+      seconds = deadline - clock.call
+      raise HttpClient::DeadlineExceeded unless seconds.positive?
+
+      Timeout.timeout(seconds, HttpClient::DeadlineExceeded, &block)
+    ensure
+      raise HttpClient::DeadlineExceeded unless deadline - clock.call > 0
+    end
+
+    # Separate from the regulatory HTTP client: this enforces the SERP host and deadline contract.
+    class HttpClient
+      REDIRECT_LIMIT = 4
+      MAX_RESPONSE_BYTES = 2 * 1024 * 1024
+      INFLATE_CHUNK_BYTES = 16 * 1024
+
+      def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, max_response_bytes: MAX_RESPONSE_BYTES,
+                     net_http: nil)
+        @clock = clock
+        @max_response_bytes = Integer(max_response_bytes)
+        @net_http = net_http || ->(uri) { Net::HTTP.new(uri.host, uri.port) }
+      end
+
+      def get(url, deadline:, allowed_hosts:)
+        fetch(URI.parse(url), deadline, allowed_hosts.map(&:downcase), REDIRECT_LIMIT)
+      rescue URI::InvalidURIError
+        HttpFailure.new(reason: "host", final_url: url)
+      end
+
+      private
+
+      attr_reader :clock, :max_response_bytes, :net_http
+
+      def fetch(uri, deadline, allowed_hosts, redirects_left)
+        return HttpFailure.new(reason: "host", final_url: uri.to_s) unless allowed_uri?(uri, allowed_hosts)
+        return HttpFailure.new(reason: "timeout", final_url: uri.to_s) unless remaining(deadline).positive?
+
+        response, body = request(uri, deadline)
+        ensure_remaining!(deadline)
+        if response.is_a?(Net::HTTPRedirection)
+          return HttpFailure.new(reason: "redirect", final_url: uri.to_s) if redirects_left.zero? || response["location"].to_s.empty?
+
+          target = begin
+            uri.merge(response["location"])
+          rescue URI::InvalidURIError
+            return HttpFailure.new(reason: "redirect", final_url: uri.to_s)
+          end
+          return HttpFailure.new(reason: "host", final_url: target.to_s) unless allowed_uri?(target, allowed_hosts)
+
+          return fetch(target, deadline, allowed_hosts, redirects_left - 1)
+        end
+        decoded = decode(body, response["content-encoding"], response["content-type"], deadline)
+        ensure_remaining!(deadline)
+
+        HttpResponse.new(status: response.code.to_i, headers: response.to_hash, body: decoded, final_url: uri.to_s)
+      rescue DeadlineExceeded, Timeout::Error
+        HttpFailure.new(reason: "timeout", final_url: uri.to_s)
+      rescue ResponseTooLarge
+        HttpFailure.new(reason: "size", final_url: uri.to_s)
+      rescue Zlib::Error
+        HttpFailure.new(reason: "parse", final_url: uri.to_s)
+      rescue SystemCallError, IOError
+        HttpFailure.new(reason: "failed", final_url: uri.to_s)
+      end
+
+      def request(uri, deadline)
+        seconds = remaining(deadline)
+        raise DeadlineExceeded unless seconds.positive?
+
+        http = net_http.call(uri)
+        http.use_ssl = uri.scheme == "https"
+        http.open_timeout = seconds
+        http.read_timeout = seconds
+        http.write_timeout = seconds
+        request = Net::HTTP::Get.new(
+          uri.request_uri.empty? ? "/" : uri.request_uri,
+          "Accept-Encoding" => "gzip, deflate",
+          "Accept-Language" => "en-US,en;q=0.9",
+          "User-Agent" => "fetch_util"
+        )
+        body = +""
+        response = within_wall_clock_timeout(seconds) do
+          http.request(request) do |incoming|
+            incoming.read_body do |chunk|
+              ensure_remaining!(deadline)
+              body << chunk
+              raise ResponseTooLarge if body.bytesize > max_response_bytes
+            end
+          end
+        end
+        [response, body]
+      end
+
+      def decode(body, encoding, content_type, deadline)
+        decoded = within_deadline(deadline) do
+          decoded = inflate(body, encoding, deadline)
+          charset = charset_from(content_type, decoded)
+          begin
+            decoded.force_encoding(charset).encode("UTF-8", invalid: :replace, undef: :replace, replace: " ")
+          rescue ArgumentError, Encoding::ConverterNotFoundError
+            decoded.force_encoding("UTF-8").scrub(" ")
+          end
+        end
+        ensure_remaining!(deadline)
+        decoded
+      end
+
+      def inflate(body, encoding, deadline)
+        return body unless %w[gzip deflate].include?(encoding.to_s.downcase)
+
+        window_bits = encoding.to_s.downcase == "gzip" ? Zlib::MAX_WBITS + 16 : Zlib::MAX_WBITS
+        inflater = Zlib::Inflate.new(window_bits)
+        decoded = +""
+        offset = 0
+        while offset < body.bytesize
+          ensure_remaining!(deadline)
+          chunk = body.byteslice(offset, INFLATE_CHUNK_BYTES)
+          append_decoded(decoded, inflater.inflate(chunk))
+          offset += chunk.bytesize
+        end
+        append_decoded(decoded, inflater.finish)
+        decoded
+      ensure
+        inflater&.close
+      end
+
+      def append_decoded(decoded, chunk)
+        decoded << chunk
+        raise ResponseTooLarge if decoded.bytesize > max_response_bytes
+      end
+
+      def charset_from(content_type, body)
+        header_charset = content_type.to_s[/charset\s*=\s*['"]?([^;'"\s]+)/i, 1]
+        return header_charset if header_charset
+
+        body.byteslice(0, 4096).to_s[/<meta\b[^>]*\bcharset\s*=\s*['"]?([^\s'";>]+)/i, 1] || "UTF-8"
+      end
+
+      def allowed_uri?(uri, allowed_hosts)
+        uri.is_a?(URI::HTTP) && uri.host && allowed_hosts.include?(uri.host.downcase)
+      end
+
+      def remaining(deadline)
+        deadline - clock.call
+      end
+
+      def ensure_remaining!(deadline)
+        raise DeadlineExceeded unless remaining(deadline).positive?
+      end
+
+      def within_wall_clock_timeout(seconds, &block)
+        return block.call unless seconds.finite?
+
+        Timeout.timeout(seconds, DeadlineExceeded, &block)
+      end
+
+      def within_deadline(deadline, &block)
+        seconds = remaining(deadline)
+        raise DeadlineExceeded unless seconds.positive?
+
+        within_wall_clock_timeout(seconds, &block)
+      end
+
+      ResponseTooLarge = Class.new(StandardError)
+      DeadlineExceeded = Class.new(StandardError)
+    end
+  end
+end