httpx 0.10.1 → 0.11.3

data/lib/httpx/plugins/multipart/mime_type_detector.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module MimeTypeDetector
+      module_function
+
+      DEFAULT_MIMETYPE = "application/octet-stream"
+
+      # inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
+      if defined?(MIME::Types)
+
+        def call(_file, filename)
+          mime = MIME::Types.of(filename).first
+          mime.content_type if mime
+        end
+
+      elsif defined?(MimeMagic)
+
+        def call(file, *)
+          mime = MimeMagic.by_magic(file)
+          mime.type if mime
+        end
+
+      elsif system("which file", out: File::NULL)
+        require "open3"
+
+        def call(file, *)
+          return if file.eof? # file command returns "application/x-empty" for empty files
+
+          Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
+            begin
+              ::IO.copy_stream(file, stdin.binmode)
+            rescue Errno::EPIPE
+            end
+            file.rewind
+            stdin.close
+
+            status = thread.value
+
+            # call to file command failed
+            if status.nil? || !status.success?
+              $stderr.print(stderr.read)
+            else
+
+              output = stdout.read.strip
+
+              if output.include?("cannot open")
+                $stderr.print(output)
+              else
+                output
+              end
+            end
+          end
+        end
+
+      else
+
+        def call(*); end
+
+      end
+    end
+  end
+end

data/lib/httpx/plugins/multipart/part.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module HTTPX
+  module Plugins::Multipart
+    module Part
+      module_function
+
+      def call(value)
+        # take out specialized objects of the way
+        if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
+          return [value, value.content_type, value.filename]
+        end
+
+        content_type = filename = nil
+
+        if value.is_a?(Hash)
+          content_type = value[:content_type]
+          filename = value[:filename]
+          value = value[:body]
+        end
+
+        value = value.open(:binmode => true) if value.is_a?(Pathname)
+
+        if value.is_a?(File)
+          filename ||= File.basename(value.path)
+          content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
+          [value, content_type, filename]
+        else
+          [StringIO.new(value.to_s), "text/plain"]
+        end
+      end
+    end
+  end
+end

data/lib/httpx/plugins/proxy/socks5.rb

@@ -159,9 +159,10 @@ module HTTPX
             packet = [VERSION, CONNECT, 0].pack("C*")
             begin
               ip = IPAddr.new(uri.host)
-              raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
 
-              packet << [IPV4, ip.to_i].pack("CN")
+              ipcode = ip.ipv6? ? IPV6 : IPV4
+
+              packet << [ipcode].pack("C") << ip.hton
             rescue IPAddr::InvalidAddressError
               packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
             end

data/lib/httpx/plugins/push_promise.rb

@@ -70,8 +70,8 @@ module HTTPX
           request.transition(:done)
           response = request.response
           response.mark_as_pushed!
-          stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
-          stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
+          stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
+          stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
         end
       end
     end

data/lib/httpx/request.rb

@@ -81,6 +81,10 @@ module HTTPX
     def response=(response)
       return unless response
 
+      if response.status == 100
+        @informational_status = response.status
+        return
+      end
       @response = response
     end
 
@@ -170,6 +174,12 @@ module HTTPX
         end
       end
 
+      def rewind
+        return if empty?
+
+        @body.rewind if @body.respond_to?(:rewind)
+      end
+
       def empty?
         return true if @body.nil?
         return false if chunked?
@@ -212,6 +222,7 @@ module HTTPX
     def transition(nextstate)
       case nextstate
       when :idle
+        @body.rewind
         @response = nil
         @drainer = nil
       when :headers
@@ -221,15 +232,15 @@ module HTTPX
                       @state == :expect
 
         if @headers.key?("expect")
-          unless @response
-            @state = :expect
-            return
-          end
-
-          case @response.status
-          when 100
-            # deallocate
-            @response = nil
+          if @informational_status && @informational_status == 100
+            # check for 100 Continue response, and deallocate the var
+            # if @informational_status == 100
+            #   @response = nil
+            # end
+          else
+            return if @state == :expect # do not re-set it
+
+            nextstate = :expect
           end
         end
       when :done
@@ -241,8 +252,7 @@ module HTTPX
     end
 
     def expects?
-      @headers["expect"] == "100-continue" &&
-        @response && @response.status == 100
+      @headers["expect"] == "100-continue" && @informational_status == 100 && !@response
     end
 
     class ProcIO

data/lib/httpx/resolver.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
 require "resolv"
-require "httpx/resolver/resolver_mixin"
-require "httpx/resolver/system"
-require "httpx/resolver/native"
-require "httpx/resolver/https"
 
 module HTTPX
   module Resolver
     extend Registry
 
+    RESOLVE_TIMEOUT = 5
+
+    require "httpx/resolver/resolver_mixin"
+    require "httpx/resolver/system"
+    require "httpx/resolver/native"
+    require "httpx/resolver/https"
+
     register :system, System
     register :native, Native
     register :https,  HTTPS

data/lib/httpx/resolver/https.rb

@@ -37,12 +37,14 @@ module HTTPX
       @connections = []
       @uri = URI(@resolver_options[:uri])
       @uri_addresses = nil
+      @resolver = Resolv::DNS.new
+      @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
     end
 
     def <<(connection)
       return if @uri.origin == connection.origin.to_s
 
-      @uri_addresses ||= Resolv.getaddresses(@uri.host)
+      @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
 
       if @uri_addresses.empty?
         ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -99,7 +101,7 @@ module HTTPX
       log { "resolver: query #{type} for #{hostname}" }
       begin
         request = build_request(hostname, type)
-        request.on(:response, &method(:on_response).curry[request])
+        request.on(:response, &method(:on_response).curry(2)[request])
         request.on(:promise, &method(:on_promise))
         @requests[request] = connection
         resolver_connection.send(request)

data/lib/httpx/resolver/native.rb

@@ -9,7 +9,6 @@ module HTTPX
     include Resolver::ResolverMixin
     using URIExtensions
 
-    RESOLVE_TIMEOUT = 5
     RECORD_TYPES = {
       "A" => Resolv::DNS::Resource::IN::A,
       "AAAA" => Resolv::DNS::Resource::IN::AAAA,
@@ -19,7 +18,7 @@ module HTTPX
       {
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     else
@@ -27,7 +26,7 @@ module HTTPX
         nameserver: nil,
         **Resolv::DNS::Config.default_config_hash,
         packet_size: 512,
-        timeouts: RESOLVE_TIMEOUT,
+        timeouts: Resolver::RESOLVE_TIMEOUT,
         record_types: RECORD_TYPES.keys,
       }.freeze
     end
@@ -148,14 +147,19 @@ module HTTPX
           queries[h] = connection
           next
         end
+
         @timeouts[host].shift
         if @timeouts[host].empty?
           @timeouts.delete(host)
           @connections.delete(connection)
-          raise NativeResolveError.new(connection, host)
+          # This loop_time passed to the exception is bogus. Ideally we would pass the total
+          # resolve timeout, including from the previous retries.
+          raise ResolveTimeoutError.new(loop_time, "Timed out")
+          # raise NativeResolveError.new(connection, host)
         else
+          log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
           connections << connection
-          log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
+          queries[h] = connection
         end
       end
       @queries = queries
@@ -279,7 +283,7 @@ module HTTPX
         @io.connect
         return unless @io.connected?
 
-        resolve if @queries.empty?
+        resolve if @queries.empty? && !@connections.empty?
       when :closed
         return unless @state == :open
 

data/lib/httpx/resolver/system.rb

@@ -20,7 +20,7 @@ module HTTPX
       timeouts = resolv_options.delete(:timeouts)
       resolv_options.delete(:cache)
       @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
-      @resolver.timeouts = timeouts if timeouts
+      @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
     end
 
     def closed?

data/lib/httpx/selector.rb

@@ -117,6 +117,7 @@ class HTTPX::Selector
     yield io
   rescue IOError, SystemCallError
     @selectables.reject!(&:closed?)
+    raise unless @selectables.empty?
   end
 
   def select(interval, &block)

data/lib/httpx/session.rb

@@ -41,7 +41,7 @@ module HTTPX
     def build_request(verb, uri, options = EMPTY_HASH)
       rklass = @options.request_class
       request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
-      request.on(:response, &method(:on_response).curry[request])
+      request.on(:response, &method(:on_response).curry(2)[request])
       request.on(:promise, &method(:on_promise))
       request
     end
@@ -136,23 +136,20 @@ module HTTPX
     def build_requests(*args, options)
       request_options = @options.merge(options)
 
-      requests = case args.size
-                 when 1
-                   reqs = args.first
-                   reqs.map do |verb, uri, opts = EMPTY_HASH|
-                     build_request(verb, uri, request_options.merge(opts))
-                   end
-                 when 2
-                   verb, uris = args
-                   if uris.respond_to?(:each)
-                     uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
-                       build_request(verb, uri, request_options.merge(opts))
-                     end
-                   else
-                     [build_request(verb, uris, request_options)]
-                   end
-                 else
-                   raise ArgumentError, "unsupported number of arguments"
+      requests = if args.size == 1
+        reqs = args.first
+        reqs.map do |verb, uri, opts = EMPTY_HASH|
+          build_request(verb, uri, request_options.merge(opts))
+        end
+      else
+        verb, uris = args
+        if uris.respond_to?(:each)
+          uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
+            build_request(verb, uri, request_options.merge(opts))
+          end
+        else
+          [build_request(verb, uris, request_options)]
+        end
       end
       raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
 

data/lib/httpx/transcoder.rb

@@ -4,18 +4,20 @@ module HTTPX
   module Transcoder
     extend Registry
 
-    def self.normalize_keys(key, value, &block)
-      if value.respond_to?(:to_ary)
+    def self.normalize_keys(key, value, cond = nil, &block)
+      if (cond && cond.call(value))
+        block.call(key.to_s, value)
+      elsif value.respond_to?(:to_ary)
         if value.empty?
           block.call("#{key}[]")
         else
           value.to_ary.each do |element|
-            normalize_keys("#{key}[]", element, &block)
+            normalize_keys("#{key}[]", element, cond, &block)
           end
         end
       elsif value.respond_to?(:to_hash)
         value.to_hash.each do |child_key, child_value|
-          normalize_keys("#{key}[#{child_key}]", child_value, &block)
+          normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
         end
       else
         block.call(key.to_s, value)

data/lib/httpx/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module HTTPX
-  VERSION = "0.10.1"
+  VERSION = "0.11.3"
 end

data/sig/connection/http2.rbs

@@ -53,12 +53,11 @@ module HTTPX
 
     def join_body: (HTTP2Next::Stream stream, Request request) -> void
 
+    def on_stream_headers: (HTTP2Next::Stream stream, Request request, Array[[String, String]] headers) -> void
 
-    # def on_stream_headers: (HTTP2Next::Stream stream, Request request, Array[String, String] headers) -> void
+    def on_stream_data: (HTTP2Next::Stream stream, Request request, string data) -> void
 
-    # def on_stream_data: (HTTP2Next::Stream stream, Request request, string data) -> void
-
-    # def on_stream_close: (HTTP2Next::Stream stream, Request request, Symbol? error) -> void
+    def on_stream_close: (HTTP2Next::Stream stream, Request request, Symbol? error) -> void
 
     def on_frame: (string bytes) -> void
 

data/sig/headers.rbs

@@ -25,6 +25,9 @@ module HTTPX
     def same_headers?: (untyped headers) -> bool
 
     def to_a: () -> Array[[headers_key, String]]
+    def to_hash: () -> Hash[headers_key, String]
+    alias to_h to_hash
+
     def inspect: () -> String 
 
     private

data/sig/plugins/multipart.rbs

@@ -3,18 +3,41 @@ module HTTPX
     module Multipart
       def self.load_dependencies: (singleton(Session)) -> void
       def self.configure: (*untyped) -> void
-      def self?.encode: (untyped) -> Encoder
+      def self?.encode: (untyped) -> (Encoder | Transcoder::Form::Encoder)
+
+      type multipart_value = string | Pathname | File | _Reader
+
+      type record_multipart_value = multipart_value |
+                                    { content_type: String, filename: String, body: multipart_value } |
+                                    { content_type: String, body: multipart_value }
+
+      type multipart_nested_value = multipart_value | _ToAry[multipart_value] | _ToHash[string, multipart_value]
+
+      type multipart_input = Enumerable[[string, multipart_value], untyped]
 
       class Encoder
         include Transcoder::_Encoder
-        include _ToS
         include _Reader
 
+        def content_type: () -> String
+
         private
 
-        def initialize: (Hash[Symbol | string, HTTP::FormData::Part | string] multipart_data) -> untyped
+        def initialize: (Hash[Symbol | string, multipart_nested_value] multipart_data) -> untyped
+
+        def header_part: (string key, String content_type, String? filename) -> StringIO
+
+        def read_chunks: (String buffer, Integer? length) -> void
+
+        def read_from_part: (Integer? max_length) -> void
+      end
+
+      module Part
+        def self?.call: (multipart_nested_value) -> ([_Reader, String, String?] | [_Reader, String])
+      end
 
-        def multipart?: (top) -> bool
+      module MimeTypeDetector
+        def self?.call: (IO file, ?String filename) -> String?
       end
     end
   end