fetch_util 0.5.2 → 0.6.0

data/lib/fetch_util/result.rb

@@ -13,86 +13,32 @@ module FetchUtil
 
     class << self
       def from_payload(url:, final_url:, payload:, canonical_url:, content_type:, warnings:, suspect:)
-        content_completeness_ratio = payload["contentCompletenessRatio"]&.to_f || 1.0
-        content_format = payload["contentFormat"]
-        paywall_state = payload["paywallState"]
         metadata = payload_metadata(
           payload,
           canonical_url: canonical_url,
           final_url: final_url,
           content_type: content_type,
           suspect: suspect,
-          warnings: warnings,
-          content_completeness_ratio: content_completeness_ratio,
-          content_format: content_format,
-          paywall_state: paywall_state,
-          name: payload["name"],
-          company: payload["company"],
-          location: payload["location"],
-          description: payload["description"],
-          ingredients: payload["ingredients"],
-          instructions: payload["instructions"],
-          bedrooms: payload["bedrooms"],
-          bathrooms: payload["bathrooms"],
-          area_sqft: payload["areaSqft"],
-          price: payload["price"],
-          rating: payload["rating"],
-          address: payload["address"],
-          social_kind: payload["socialKind"],
-          platform: payload["platform"],
-          handle: payload["handle"],
-          reply_count: payload["replyCount"],
-          community: payload["community"],
-          score: payload["score"]
+          warnings: warnings
         )
 
         new(
           url: url,
           final_url: final_url,
-          title: payload["title"],
-          byline: payload["byline"],
-          excerpt: payload["excerpt"],
-          site_name: payload["siteName"],
-          published_time: payload["publishedTime"],
-          canonical_url: canonical_url,
-          language: payload["language"],
-          name: payload["name"],
-          company: payload["company"],
-          location: payload["location"],
-          description: payload["description"],
-          ingredients: payload["ingredients"],
-          instructions: payload["instructions"],
-          bedrooms: payload["bedrooms"],
-          bathrooms: payload["bathrooms"],
-          area_sqft: payload["areaSqft"],
           html: payload["html"],
           markdown: payload["markdown"],
           metadata: metadata,
-          reader_mode: payload["readerMode"],
-          content_type: content_type,
-          suspect: suspect,
-          warnings: warnings,
-          content_completeness_ratio: content_completeness_ratio,
-          content_format: content_format,
-          paywall_state: paywall_state,
-          price: payload["price"],
-          rating: payload["rating"],
-          address: payload["address"],
-          social_kind: payload["socialKind"],
-          platform: payload["platform"],
-          handle: payload["handle"],
-          reply_count: payload["replyCount"],
-          community: payload["community"],
-          score: payload["score"]
+          **metadata.except(:content_url)
         )
       end
 
       def error(url:, warning:, message:)
+        warnings = [warning]
         metadata = {
           content_url: url,
           content_type: "error",
           suspect: true,
-          warnings: [warning],
+          warnings: warnings,
           error_message: message,
           social_kind: nil,
           platform: nil,
@@ -127,18 +73,14 @@ module FetchUtil
           reader_mode: nil,
           content_type: "error",
           suspect: true,
-          warnings: [warning],
+          warnings: warnings,
           error_message: message
         )
       end
 
       private
 
-      def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:,
-                           content_completeness_ratio:, content_format:, paywall_state:,
-                           name:, company:, location:, description:, ingredients:, instructions:,
-                           bedrooms:, bathrooms:, area_sqft:, price:, rating:, address:,
-                           social_kind:, platform:, handle:, reply_count:, community:, score:)
+      def payload_metadata(payload, canonical_url:, final_url:, content_type:, suspect:, warnings:)
         {
           title: payload["title"],
           byline: payload["byline"],
@@ -147,32 +89,32 @@ module FetchUtil
           published_time: payload["publishedTime"],
           canonical_url: canonical_url,
           language: payload["language"],
-          name: name,
-          company: company,
-          location: location,
-          description: description,
-          ingredients: ingredients,
-          instructions: instructions,
-          bedrooms: bedrooms,
-          bathrooms: bathrooms,
-          area_sqft: area_sqft,
+          name: payload["name"],
+          company: payload["company"],
+          location: payload["location"],
+          description: payload["description"],
+          ingredients: payload["ingredients"],
+          instructions: payload["instructions"],
+          bedrooms: payload["bedrooms"],
+          bathrooms: payload["bathrooms"],
+          area_sqft: payload["areaSqft"],
           content_url: final_url,
           reader_mode: payload["readerMode"],
           content_type: content_type,
           suspect: suspect,
           warnings: warnings,
-          content_completeness_ratio: content_completeness_ratio,
-          content_format: content_format,
-          paywall_state: paywall_state,
-          price: price,
-          rating: rating,
-          address: address,
-          social_kind: social_kind,
-          platform: platform,
-          handle: handle,
-          reply_count: reply_count,
-          community: community,
-          score: score
+          content_completeness_ratio: payload["contentCompletenessRatio"]&.to_f || 1.0,
+          content_format: payload["contentFormat"],
+          paywall_state: payload["paywallState"],
+          price: payload["price"],
+          rating: payload["rating"],
+          address: payload["address"],
+          social_kind: payload["socialKind"],
+          platform: payload["platform"],
+          handle: payload["handle"],
+          reply_count: payload["replyCount"],
+          community: payload["community"],
+          score: payload["score"]
         }.freeze
       end
     end
@@ -184,6 +126,12 @@ module FetchUtil
                    content_completeness_ratio: 1.0, content_format: nil, paywall_state: nil,
                    price: nil, rating: nil, address: nil, social_kind: nil, platform: nil,
                    handle: nil, reply_count: nil, community: nil, score: nil, error_message: nil)
+      owned_warnings = warnings.map { |warning| warning.to_s.dup.freeze }.freeze
+      owned_content_format = content_format&.dup&.freeze
+      owned_paywall_state = paywall_state&.dup&.freeze
+      owned_metadata = metadata.merge(warnings: owned_warnings)
+      owned_metadata[:content_format] = owned_content_format if owned_metadata.key?(:content_format)
+      owned_metadata[:paywall_state] = owned_paywall_state if owned_metadata.key?(:paywall_state)
       @url = url
       @final_url = final_url
       @title = title
@@ -204,14 +152,14 @@ module FetchUtil
       @area_sqft = area_sqft
       @html = html
       @markdown = markdown
-      @metadata = metadata.freeze
+      @metadata = owned_metadata.freeze
       @reader_mode = reader_mode
       @content_type = content_type
       @suspect = suspect
-      @warnings = warnings.freeze
+      @warnings = owned_warnings
       @content_completeness_ratio = content_completeness_ratio
-      @content_format = content_format&.freeze
-      @paywall_state = paywall_state&.freeze
+      @content_format = owned_content_format
+      @paywall_state = owned_paywall_state
       @price = price
       @rating = rating
       @address = address

data/lib/fetch_util/search_transport.rb

@@ -13,22 +13,47 @@ module FetchUtil
   class SearchTransport
     Candidate = Data.define(:source, :title, :url, :snippet, :source_rank) do
       def initialize(source:, title:, url:, source_rank:, snippet: nil)
-        super(source: source.to_s.freeze, title: title.to_s.freeze, url: url.to_s.freeze,
-              snippet: snippet&.to_s&.freeze, source_rank: Integer(source_rank))
+        super(source: source.to_s.dup.freeze, title: title.to_s.dup.freeze, url: url.to_s.dup.freeze,
+              snippet: snippet.nil? ? nil : snippet.to_s.dup.freeze, source_rank: Integer(source_rank))
       end
     end
 
     SourceResponse = Data.define(:source, :transport, :status, :candidates, :elapsed_ms, :final_url, :reason) do
       def initialize(source:, status:, elapsed_ms:, transport: "http", candidates: [], final_url: nil, reason: nil)
-        super(source: source.to_s.freeze, transport: transport.to_s.freeze, status: status.to_s.freeze,
-              candidates: candidates.freeze, elapsed_ms: Integer(elapsed_ms), final_url: final_url&.to_s&.freeze,
-              reason: reason&.to_s&.freeze)
+        super(source: source.to_s.dup.freeze, transport: transport.to_s.dup.freeze, status: status.to_s.dup.freeze,
+              candidates: candidates.dup.freeze, elapsed_ms: Integer(elapsed_ms),
+              final_url: final_url.nil? ? nil : final_url.to_s.dup.freeze,
+              reason: reason.nil? ? nil : reason.to_s.dup.freeze)
       end
     end
 
     HttpResponse = Data.define(:status, :headers, :body, :final_url)
     HttpFailure = Data.define(:reason, :final_url)
 
+    class MonotonicClock
+      def initialize(source)
+        @source = source
+        @mutex = Mutex.new
+        @latest = nil
+      end
+
+      def call
+        @mutex.synchronize do
+          current = @source.call
+          begin
+            current = Float(current)
+          rescue ArgumentError, TypeError
+            raise ArgumentError, "clock must return a finite number"
+          end
+          raise ArgumentError, "clock must return a finite number" unless current.finite?
+
+          @latest = current if @latest.nil? || current > @latest
+          @latest
+        end
+      end
+    end
+    private_constant :MonotonicClock
+
     SOURCES = {
       "brave" => { url: "https://search.brave.com/search?q=%{query}", hosts: %w[search.brave.com] },
       "bing" => { url: "https://www.bing.com/search?q=%{query}&setlang=en-US&cc=US", hosts: %w[www.bing.com cn.bing.com] },
@@ -55,15 +80,18 @@ module FetchUtil
     }.freeze
 
     def initialize(sources: SOURCES.keys, timeout: DEFAULT_TIMEOUT, clock: nil, http_client: nil, html_parser: nil)
-      @sources = sources.map(&:to_s).freeze
+      @sources = sources.map { |source| source.to_s.dup.freeze }.freeze
       unknown = @sources - SOURCES.keys
-      raise ArgumentError, "unknown search sources: #{unknown.join(", ")}" if unknown.any?
+      raise InputError, "unknown search sources: #{unknown.join(", ")}" if unknown.any?
 
-      @timeout = Float(timeout)
-      raise ArgumentError, "timeout must be positive" unless @timeout.positive?
+      @timeout = validated_timeout(timeout)
 
-      @clock = clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
-      @http_client = http_client || HttpClient.new(clock: @clock)
+      @clock = MonotonicClock.new(clock || -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+      @http_client = if http_client.is_a?(HttpClient)
+                       http_client.send(:with_clock, @clock)
+                     else
+                       http_client || HttpClient.new(clock: @clock)
+                     end
       @html_parser = html_parser || ->(body) { Nokogiri::HTML(body) }
     end
 
@@ -83,15 +111,20 @@ module FetchUtil
 
     def search(query, timeout: @timeout)
       query = query.to_s.strip
-      raise ArgumentError, "query must not be empty" if query.empty?
+      raise InputError, "query must not be empty" if query.empty?
 
-      request_timeout = Float(timeout)
-      raise ArgumentError, "timeout must be positive" unless request_timeout.positive?
+      request_timeout = validated_timeout(timeout)
 
       deadline = clock.call + request_timeout
       responses = Array.new(sources.length)
-      threads = sources.each_with_index.map do |source, index|
-        Thread.new { responses[index] = search_source(source, query, deadline) }
+      threads = []
+      begin
+        sources.each_with_index do |source, index|
+          threads << Thread.new { responses[index] = search_source(source, query, deadline) }
+        end
+      rescue ThreadError
+        threads.each(&:join)
+        raise
       end
       threads.each(&:join)
       responses
@@ -101,6 +134,13 @@ module FetchUtil
 
     attr_reader :clock, :html_parser, :http_client, :sources, :timeout
 
+    def validated_timeout(value)
+      timeout = Float(value)
+      raise InputError, "timeout must be positive" unless timeout.finite? && timeout.positive?
+
+      timeout
+    end
+
     def search_source(source, query, deadline)
       started_at = clock.call
       url = build_url(source, query)
@@ -110,9 +150,10 @@ module FetchUtil
       end
       outcome = within_deadline(deadline) do
         document = html_parser.call(result.body)
-        next [:failed, "challenge"] if challenge?(source, result, document)
+        shell_text = search_shell_text(source, document)
+        next [:failed, "challenge"] if challenge?(source, result, shell_text)
         next [:failed, "http_status"] unless result.status.between?(200, 299)
-        next [:empty] if no_results?(source, document)
+        next [:empty] if no_results?(source, shell_text)
 
         candidates = parse_candidates(source, document)
         next [:ok, candidates] if candidates.empty?
@@ -359,19 +400,16 @@ module FetchUtil
       node ? node.text.encode("UTF-8", invalid: :replace, undef: :replace, replace: " ").gsub(/\s+/, " ").strip : ""
     end
 
-    def challenge?(source, response, document)
+    def challenge?(source, response, shell_text)
       return true if source == "google" && URI.parse(response.final_url).path.start_with?("/sorry")
       return true if source == "duckduckgo" && response.status == 202
 
-      title = normalized_text(document.at_css("title")).downcase
-      visible_text = visible_document_text(document)
-      "#{title} #{visible_text}".match?(/captcha|unusual traffic|verify you are human|consent/)
+      shell_text.match?(/captcha|unusual traffic|verify you are human|consent/)
     rescue URI::InvalidURIError
       true
     end
 
-    def no_results?(source, document)
-      text = visible_document_text(document)
+    def no_results?(source, text)
       patterns = {
         "brave" => /no results|did not match any documents/,
         "bing" => /there are no results|no results found/,
@@ -383,9 +421,10 @@ module FetchUtil
       text.match?(patterns.fetch(source))
     end
 
-    def visible_document_text(document)
+    def search_shell_text(source, document)
       visible = document.dup
-      visible.css("script, style, template, noscript").remove
+      result_nodes(source, visible).each(&:remove)
+      visible.css("title, script, style, template, noscript").remove
       normalized_text(visible).downcase
     end
 
@@ -427,8 +466,8 @@ module FetchUtil
 
       def initialize(clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }, max_response_bytes: MAX_RESPONSE_BYTES,
                      net_http: nil)
-        @clock = clock
-        @max_response_bytes = Integer(max_response_bytes)
+        @clock = clock.is_a?(MonotonicClock) ? clock : MonotonicClock.new(clock)
+        @max_response_bytes = positive_max_response_bytes(max_response_bytes)
         @net_http = net_http || ->(uri) { Net::HTTP.new(uri.host, uri.port) }
       end
 
@@ -442,6 +481,21 @@ module FetchUtil
 
       attr_reader :clock, :max_response_bytes, :net_http
 
+      def with_clock(shared_clock)
+        copy = dup
+        copy.instance_variable_set(:@clock, shared_clock)
+        copy
+      end
+
+      def positive_max_response_bytes(value)
+        bytes = Integer(value)
+        return bytes if bytes.positive?
+
+        raise ArgumentError
+      rescue ArgumentError, TypeError, FloatDomainError
+        raise InputError, "max_response_bytes must be positive"
+      end
+
       def fetch(uri, deadline, allowed_hosts, redirects_left)
         return HttpFailure.new(reason: "host", final_url: uri.to_s) unless allowed_uri?(uri, allowed_hosts)
         return HttpFailure.new(reason: "timeout", final_url: uri.to_s) unless remaining(deadline).positive?
@@ -456,6 +510,9 @@ module FetchUtil
           rescue URI::InvalidURIError
             return HttpFailure.new(reason: "redirect", final_url: uri.to_s)
           end
+          if uri.scheme == "https" && target.scheme != "https"
+            return HttpFailure.new(reason: "redirect", final_url: target.to_s)
+          end
           return HttpFailure.new(reason: "host", final_url: target.to_s) unless allowed_uri?(target, allowed_hosts)
 
           return fetch(target, deadline, allowed_hosts, redirects_left - 1)
@@ -470,7 +527,7 @@ module FetchUtil
         HttpFailure.new(reason: "size", final_url: uri.to_s)
       rescue Zlib::Error
         HttpFailure.new(reason: "parse", final_url: uri.to_s)
-      rescue SystemCallError, IOError
+      rescue IOError, SocketError, SystemCallError
         HttpFailure.new(reason: "failed", final_url: uri.to_s)
       end
 
@@ -494,8 +551,7 @@ module FetchUtil
           http.request(request) do |incoming|
             incoming.read_body do |chunk|
               ensure_remaining!(deadline)
-              body << chunk
-              raise ResponseTooLarge if body.bytesize > max_response_bytes
+              append_bounded(body, chunk)
             end
           end
         end
@@ -526,18 +582,29 @@ module FetchUtil
         while offset < body.bytesize
           ensure_remaining!(deadline)
           chunk = body.byteslice(offset, INFLATE_CHUNK_BYTES)
-          append_decoded(decoded, inflater.inflate(chunk))
+          inflater.inflate(chunk) do |piece|
+            ensure_remaining!(deadline)
+            append_decoded(decoded, piece)
+          end
           offset += chunk.bytesize
         end
-        append_decoded(decoded, inflater.finish)
+        inflater.finish do |piece|
+          ensure_remaining!(deadline)
+          append_decoded(decoded, piece)
+        end
         decoded
       ensure
         inflater&.close
       end
 
       def append_decoded(decoded, chunk)
-        decoded << chunk
-        raise ResponseTooLarge if decoded.bytesize > max_response_bytes
+        append_bounded(decoded, chunk)
+      end
+
+      def append_bounded(buffer, chunk)
+        raise ResponseTooLarge if buffer.bytesize + chunk.bytesize > max_response_bytes
+
+        buffer << chunk
       end
 
       def charset_from(content_type, body)

data/lib/fetch_util/searcher.rb

@@ -16,9 +16,9 @@ module FetchUtil
                    timeout: SearchTransport::DEFAULT_TIMEOUT)
       @request_log = request_log
       @sources_explicit = !sources.nil?
-      @sources = Array(sources || DEFAULT_SOURCES).map(&:to_s).uniq
+      @sources = Array(sources || DEFAULT_SOURCES).map { |source| source.to_s.dup.freeze }.uniq.freeze
       unknown = @sources - SearchTransport::SOURCES.keys
-      raise ArgumentError, "unsupported search source: #{unknown.first}" if unknown.any?
+      raise InputError, "unsupported search source: #{unknown.first}" if unknown.any?
 
       validate_limit!(limit)
       @limit = limit
@@ -28,16 +28,18 @@ module FetchUtil
 
     def search(query)
       encoded_query = query.to_s.strip
-      raise ArgumentError, "query must not be empty" if encoded_query.empty?
+      raise InputError, "query must not be empty" if encoded_query.empty?
 
       @request_log.append(search_request_uri(encoded_query))
       responses = @transport.search(encoded_query)
+      response_index = {}
+      responses.each { |response| response_index[response.source] ||= response }
 
       payload = {
         query: encoded_query,
-        results: formatted_results(apply_limit(aggregate(responses, encoded_query)))
+        results: formatted_results(apply_limit(aggregate(response_index, encoded_query)))
       }
-      payload[:diagnostics] = diagnostics(responses) if @verbose
+      payload[:diagnostics] = diagnostics(response_index) if @verbose
       payload
     end
 
@@ -53,23 +55,23 @@ module FetchUtil
       return if value.nil?
       return if value.is_a?(Integer) && value >= 0
 
-      raise ArgumentError, "limit must be a nonnegative integer"
+      raise InputError, "limit must be a nonnegative integer"
     end
 
     def search_request_uri(query)
       "search://#{@sources.join(",")}?q=#{CGI.escape(query)}"
     end
 
-    def aggregate(responses, query)
+    def aggregate(response_index, query)
       parsed = {}
       structured_query = query.match?(STRUCTURED_QUERY)
 
       @sources.each do |source|
-        response = responses.find { |item| item.source == source }
+        response = response_index[source]
         parsed[source] = response ? response.candidates.filter_map { |candidate| normalized_candidate(candidate) } : []
       end
 
-      apply_structured_query_authority!(parsed, responses) if structured_query
+      apply_structured_query_authority!(parsed, response_index) if structured_query
       max_size = parsed.values.map(&:length).max || 0
 
       items = []
@@ -95,11 +97,11 @@ module FetchUtil
       items
     end
 
-    def apply_structured_query_authority!(parsed, responses)
+    def apply_structured_query_authority!(parsed, response_index)
       return if @sources_explicit
       return unless @sources.include?(STRUCTURED_QUERY_AUTHORITY)
 
-      authority = responses.find { |response| response.source == STRUCTURED_QUERY_AUTHORITY }
+      authority = response_index[STRUCTURED_QUERY_AUTHORITY]
       return unless authority&.status == "ok" && authority.reason.nil?
 
       authority_urls = parsed.fetch(STRUCTURED_QUERY_AUTHORITY).map { |item| item[:url] }
@@ -146,9 +148,9 @@ module FetchUtil
       end
     end
 
-    def diagnostics(responses)
+    def diagnostics(response_index)
       @sources.filter_map do |source|
-        response = responses.find { |item| item.source == source }
+        response = response_index[source]
         next unless response
 
         diagnostic = {

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.5.2"
+  VERSION = "0.6.0"
 end

data/lib/fetch_util.rb

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
 require "addressable/uri"
+require "ferrum"
+require "public_suffix"
 require "uri"
 
 require_relative "fetch_util/version"
 
 module FetchUtil
   class Error < StandardError; end
+  class InputError < ArgumentError; end
   class BrowserError < Error; end
   class ExtractionError < Error; end
 
@@ -69,7 +72,7 @@ module FetchUtil
   module_function
 
   def fetch(url, **options)
-    Fetcher.new(**options).fetch(url)
+    Fetcher.fetch_once(url, **options)
   end
 
   def fetch_many(urls, **options)