fetch_util 0.3.1 → 0.3.2

data/lib/fetch_util/parallel_fetcher.rb

@@ -38,11 +38,11 @@ module FetchUtil
       work = Array(urls).compact.map(&:to_s).reject(&:empty?)
       return [] if work.empty?
 
-      jobs = Queue.new
-      failures = Queue.new
-      work.each_with_index { |url, index| jobs << [index, url] }
       results = Array.new(work.length)
       worker_count = [@concurrency, work.length].min
+      failures = []
+      next_index = 0
+      mutex = Mutex.new
 
       threads = Array.new(worker_count) do
         Thread.new do
@@ -50,44 +50,39 @@ module FetchUtil
 
           begin
             loop do
-              begin
-                index, url = jobs.pop(true)
-              rescue ThreadError
-                break
+              index = mutex.synchronize do
+                if next_index < work.length
+                  current = next_index
+                  next_index += 1
+                  current
+                end
               end
+              break if index.nil?
+
+              url = work[index]
 
               begin
                 results[index] = fetcher.fetch(url)
               rescue StandardError => e
-                failures << Failure.new(index: index, url: url, error: e)
+                mutex.synchronize { failures << Failure.new(index: index, url: url, error: e) }
               end
             end
           ensure
             fetcher.quit if fetcher.respond_to?(:quit)
           end
         rescue StandardError => e
-          failures << Failure.new(index: nil, url: nil, error: e)
+          mutex.synchronize { failures << Failure.new(index: nil, url: nil, error: e) }
         end
       end
 
       threads.each(&:join)
-      raise_for_failures(drain_queue(failures), results)
+      raise_for_failures(failures, results)
 
       results
     end
 
     private
 
-    def drain_queue(queue)
-      items = []
-      loop do
-        items << queue.pop(true)
-      rescue ThreadError
-        break
-      end
-      items
-    end
-
     def raise_for_failures(failures, results)
       return if failures.empty?
 

data/lib/fetch_util/raw_docs_fallback.rb

@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
 require "cgi"
-require "net/http"
 require "nokogiri"
 require "uri"
 
+require_relative "regulatory/http_client"
+
 module FetchUtil
   class RawDocsFallback
     DEFAULT_HEADERS = {
@@ -47,8 +48,9 @@ module FetchUtil
     ].freeze
     PRUNED_TEXT_PATTERN = /\A(?:on this page|table of contents|edit this page|copy page|copy item path|search|settings|help|expand description)\z/i
 
-    def initialize(timeout: 20)
+    def initialize(timeout: 20, http_client: nil)
       @timeout = timeout.to_i
+      @http_client = http_client || FetchUtil::HttpRedirectClient.new(timeout: @timeout, headers: DEFAULT_HEADERS)
     end
 
     def fetch(url)
@@ -272,26 +274,11 @@ module FetchUtil
       %(concat(#{parts.join(%q(, "'", ))}))
     end
 
-    def fetch_html(url, limit: 5)
-      raise URI::InvalidURIError, "too many redirects" if limit <= 0
-
-      uri = URI.parse(url)
-      request = Net::HTTP::Get.new(uri)
-      DEFAULT_HEADERS.each { |key, value| request[key] = value }
-
-      response = Net::HTTP.start(uri.host, uri.port, use_ssl: uri.scheme == "https", open_timeout: @timeout, read_timeout: @timeout) do |http|
-        http.request(request)
-      end
+    def fetch_html(url)
+      response = @http_client.get(url)
+      raise Error, "HTTP #{response.status}" unless response.status.between?(200, 299)
 
-      case response
-      when Net::HTTPSuccess
-        [uri.to_s, response.body]
-      when Net::HTTPRedirection
-        location = URI.join(uri, response["location"]).to_s
-        fetch_html(location, limit: limit - 1)
-      else
-        raise Error, "HTTP #{response.code}"
-      end
+      [response.url, response.body]
     end
   end
 end

data/lib/fetch_util/regulatory/http_client.rb

@@ -1,70 +1,138 @@
 # frozen_string_literal: true
 
 require "net/http"
+require "uri"
 
 module FetchUtil
-  class Regulatory
-    class HttpClient
-      REDIRECT_LIMIT = 5
+  class HttpRedirectClient
+    REDIRECT_LIMIT = 5
+    TRANSIENT_ERRORS = [EOFError, IOError, SocketError, SystemCallError, Timeout::Error].freeze
+    Response = Struct.new(:url, :status, :headers, :body, :redirects, keyword_init: true)
 
-      def initialize(timeout:, user_agent:)
-        @timeout = timeout.to_f
-        @user_agent = user_agent.to_s.strip
-      end
+    def initialize(timeout:, headers: {})
+      @timeout = timeout.to_f
+      @headers = headers.reject { |_key, value| value.to_s.empty? }
+    end
 
-      def get(url, limit: REDIRECT_LIMIT)
-        uri = parse_http_uri(url)
-        fetch(uri, limit, [])
-      end
+    def get(url, limit: REDIRECT_LIMIT)
+      @connections = {}
+      fetch(parse_http_uri(url), limit, [])
+    ensure
+      close_connections
+    end
 
-      private
+    private
 
-      attr_reader :timeout, :user_agent
+    attr_reader :timeout, :headers
 
-      def fetch(uri, limit, redirects)
-        response = request(uri)
-        return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
+    def fetch(uri, limit, redirects)
+      response = request(uri)
+      return build_response(uri, response, redirects: redirects) unless response.is_a?(Net::HTTPRedirection)
 
-        raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
+      raise FetchUtil::Error, "too many redirects for #{uri}" if limit <= 0
 
-        location = response["location"].to_s.strip
-        return build_response(uri, response, redirects: redirects) if location.empty?
+      location = response["location"].to_s.strip
+      return build_response(uri, response, redirects: redirects) if location.empty?
 
-        redirect_response = build_response(uri, response)
-        fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
-      end
+      redirect_response = build_response(uri, response)
+      fetch(uri.merge(location), limit - 1, redirects + [redirect_response])
+    end
 
-      def request(uri)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = uri.scheme == "https"
-        http.open_timeout = timeout
-        http.read_timeout = timeout
+    def request(uri)
+      attempts = 0
+      begin
+        http = connection_for(uri)
         request = Net::HTTP::Get.new(uri.request_uri.empty? ? "/" : uri.request_uri)
-        request["Accept"] = "text/html,application/json,text/plain,*/*"
-        request["User-Agent"] = user_agent unless user_agent.empty?
+        headers.each { |key, value| request[key] = value }
         http.request(request)
+      rescue *TRANSIENT_ERRORS
+        close_connection(uri)
+        attempts += 1
+        retry if attempts <= 1
+        raise
       end
+    end
 
-      def build_response(uri, response, redirects: [])
-        FetchUtil::Regulatory::Response.new(
-          url: uri.to_s,
-          status: response.code.to_i,
-          headers: response.to_hash.transform_keys(&:downcase),
-          body: response.body.to_s,
-          redirects: redirects
-        )
+    def connection_for(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections[key] ||= Net::HTTP.start(
+        uri.host,
+        uri.port,
+        use_ssl: uri.scheme == "https",
+        open_timeout: timeout,
+        read_timeout: timeout
+      )
+    end
+
+    def close_connection(uri)
+      key = [uri.scheme, uri.host, uri.port]
+      @connections.delete(key)&.finish
+    rescue IOError
+      nil
+    end
+
+    def close_connections
+      @connections&.each_value do |http|
+        http.finish if http.started?
+      rescue IOError
+        nil
       end
+      @connections = nil
+    end
+
+    def build_response(uri, response, redirects: [])
+      Response.new(
+        url: uri.to_s,
+        status: response.code.to_i,
+        headers: response.to_hash.transform_keys(&:downcase),
+        body: response.body.to_s,
+        redirects: redirects
+      )
+    end
+
+    def parse_http_uri(url)
+      uri = URI.parse(url.to_s)
+      raise URI::InvalidURIError, "unsupported url: #{url}" unless uri.is_a?(URI::HTTP) && uri.host
+
+      uri
+    end
+  end
 
-      def parse_http_uri(url)
-        uri = URI.parse(url.to_s)
-        unless uri.is_a?(URI::HTTP) && uri.host
-          raise ArgumentError, "unsupported url: #{url}"
-        end
+  class Regulatory
+    class HttpClient
+      DEFAULT_ACCEPT = "text/html,application/json,text/plain,*/*"
+
+      def initialize(timeout:, user_agent:, redirect_client: nil)
+        @user_agent = user_agent.to_s.strip
+        @redirect_client = redirect_client || FetchUtil::HttpRedirectClient.new(timeout: timeout, headers: request_headers)
+      end
 
-        uri
+      def get(url, limit: FetchUtil::HttpRedirectClient::REDIRECT_LIMIT)
+        build_response(redirect_client.get(url, limit: limit))
       rescue URI::InvalidURIError
         raise ArgumentError, "unsupported url: #{url}"
       end
+
+      private
+
+      attr_reader :redirect_client, :user_agent
+
+      def build_response(response)
+        FetchUtil::Regulatory::Response.new(
+          url: response.url,
+          status: response.status,
+          headers: response.headers,
+          body: response.body,
+          redirects: response.redirects.map { |redirect| build_response(redirect) }
+        )
+      end
+
+      def request_headers
+        {
+          "Accept" => DEFAULT_ACCEPT,
+          "User-Agent" => user_agent
+        }
+      end
     end
   end
 end

data/lib/fetch_util/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FetchUtil
-  VERSION = "0.3.1"
+  VERSION = "0.3.2"
 end

data/lib/fetch_util.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "addressable/uri"
 require "uri"
 
 require_relative "fetch_util/version"
@@ -88,12 +89,21 @@ module FetchUtil
     text.gsub(/\u00A0/, " ").gsub(/\s+/, " ").strip
   end
 
+  def normalize_url(url)
+    return url if url.nil? || url.to_s.empty?
+
+    Addressable::URI.parse(url.to_s).normalize.to_s
+  rescue Addressable::URI::InvalidURIError, URI::InvalidURIError
+    url.to_s
+  end
+
   def strip_www_host(url)
-    URI.parse(url.to_s).host.to_s.downcase.sub(/\Awww\./, "")
+    uri = url.is_a?(URI::Generic) ? url : URI.parse(url.to_s)
+    uri.host.to_s.downcase.sub(/\Awww\./, "")
   end
 
   def docs_like_url?(value)
-    uri = value.is_a?(URI::Generic) ? value : URI.parse(value.to_s.strip)
+    uri = value.is_a?(URI::Generic) ? value : URI.parse(normalize_url(value.to_s.strip))
     return false unless uri.is_a?(URI::HTTP) && uri.host
 
     host = strip_www_host(uri)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fetch_util
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.2
 platform: ruby
 authors:
 - hmdne
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.15
 specification_version: 4
 summary: AI for fetching in Ruby
 test_files: []