ethon 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 87e1825a32b82c558aded416768e7dffc8a1be6918bca888187b343e17159432
-  data.tar.gz: 4773483d3d4e5bf46298d55c4d4967622666bf28f6665445d621ab2b4de30213
+  metadata.gz: 04b0080762fbe70bea3b3d5717749a8afb60feb1d843543fc313705dc245b2b8
+  data.tar.gz: fab2787f5a0eebd10ed2d6004ba9b6e3cdc5b60f776e9e5ee0d811529a9a50c6
 SHA512:
-  metadata.gz: bc51f162c86e89ddc1176368a508870e0be217a11027ae17d93149cbc04ae822d2d03ce6e2fa0e15269bbafefbda9e2dbfd1e64aba0b4e3fddbf820b958a3b82
-  data.tar.gz: 8c755c396544992d6062fe28f3a7a741c3269941665e89995a5f5060d08b4ed8d4f5768bc0601efa7780b1407c49d1839e4557f3ac591abc0529438ad8506163
+  metadata.gz: c8d9d0214408c48e702b0d8256d5d6c9d62540d57b9ce142c4a90ff667bbf4db4661e9423ce372d0652c9df4b0147110c3dcc2f317418d3b0e33fe245b4b876b
+  data.tar.gz: 1d351ae5e4b45e736a7eef6d4068cec779bfba6a71097c102a3998315aa391789d86725b173eafc5f15d55e587730334ec6c1eb60d4c5eb63b3cacbd11ddddb1

data/CHANGELOG.md

@@ -2,7 +2,11 @@
 
 ## Master
 
-[Full Changelog](https://github.com/typhoeus/ethon/compare/v0.12.0...master)
+[Full Changelog](https://github.com/typhoeus/ethon/compare/v0.15.0...master)
+
+## 0.15.0
+
+[Full Changelog](https://github.com/typhoeus/ethon/compare/v0.14.0...v0.15.0)
 
 ## 0.12.0
 

data/lib/ethon/curls/constants.rb

@@ -59,5 +59,22 @@ module Ethon
     VERSION_NTLM_WB = (1<<15) # NTLM delegating to winbind helper
     VERSION_HTTP2 = (1<<16) # HTTP2 support built
     VERSION_GSSAPI = (1<<17) # GSS-API is supported
+
+    SOCKET_BAD = -1
+    SOCKET_TIMEOUT = SOCKET_BAD
+
+    PollAction = enum(:poll_action, [
+      :none,
+      :in,
+      :out,
+      :inout,
+      :remove
+    ])
+
+    SocketReadiness = bitmask(:socket_readiness, [
+      :in,  # CURL_CSELECT_IN  - 0x01 (bit 0)
+      :out, # CURL_CSELECT_OUT - 0x02 (bit 1)
+      :err, # CURL_CSELECT_ERR - 0x04 (bit 2)
+    ])
   end
 end

data/lib/ethon/curls/functions.rb

@@ -36,6 +36,7 @@ module Ethon
         base.attach_function :multi_fdset,                :curl_multi_fdset,         [:pointer, Curl::FDSet.ptr, Curl::FDSet.ptr, Curl::FDSet.ptr, :pointer], :multi_code
         base.attach_function :multi_strerror,             :curl_multi_strerror,      [:int],                         :string
         base.attach_function :multi_setopt,               :curl_multi_setopt,        [:pointer, :multi_option, :varargs], :multi_code
+        base.attach_function :multi_socket_action,        :curl_multi_socket_action, [:pointer, :int, :socket_readiness, :pointer], :multi_code
 
         base.attach_function :version,                    :curl_version,             [],                             :string
         base.attach_function :version_info,               :curl_version_info,        [],                             Curl::VersionInfoData.ptr

data/lib/ethon/curls/options.rb

@@ -82,6 +82,12 @@ module Ethon
         when :callback
           va_type=:callback
           raise Errors::InvalidValue.new(option,value) unless value.nil? or value.is_a? Proc
+        when :socket_callback
+          va_type=:socket_callback
+          raise Errors::InvalidValue.new(option,value) unless value.nil? or value.is_a? Proc
+        when :timer_callback
+          va_type=:timer_callback
+          raise Errors::InvalidValue.new(option,value) unless value.nil? or value.is_a? Proc
         when :debug_callback
           va_type=:debug_callback
           raise Errors::InvalidValue.new(option,value) unless value.nil? or value.is_a? Proc
@@ -137,6 +143,8 @@ module Ethon
         :dontuse_object => :objectpoint, # An object we don't support (e.g. FILE*)
         :cbdata => :objectpoint,
         :callback => :functionpoint,
+        :socket_callback => :functionpoint,
+        :timer_callback => :functionpoint,
         :debug_callback => :functionpoint,
         :progress_callback => :functionpoint,
         :off_t => :off_t,
@@ -208,10 +216,10 @@ module Ethon
       # Documentation @ http://curl.haxx.se/libcurl/c/curl_multi_setopt.html
       option_type :multi
 
-      option :multi, :socketfunction, :callback, 1
+      option :multi, :socketfunction, :socket_callback, 1
       option :multi, :socketdata, :cbdata, 2
       option :multi, :pipelining, :int, 3
-      option :multi, :timerfunction, :callback, 4
+      option :multi, :timerfunction, :timer_callback, 4
       option :multi, :timerdata, :cbdata, 5
       option :multi, :maxconnects, :int, 6
       option :multi, :max_host_connections, :int, 7
@@ -450,6 +458,29 @@ module Ethon
       option :easy, :gssapi_delegation, :bitmask, 210, [:none, :policy_flag, :flag]
       option :easy, :pinnedpublickey, :string, 230
       option_alias :easy, :pinnedpublickey, :pinned_public_key
+      ## PROXY SSL OPTIONS
+      option :easy, :proxy_cainfo, :string, 246
+      option :easy, :proxy_capath, :string, 247
+      option :easy, :proxy_ssl_verifypeer, :bool, 248
+      option :easy, :proxy_ssl_verifyhost, :int, 249
+      option :easy, :proxy_sslversion, :enum, 250, [:default, :tlsv1, :sslv2, :sslv3, :tlsv1_0, :tlsv1_1, :tlsv1_2]
+      option :easy, :proxy_tlsauth_username, :string, 251
+      option :easy, :proxy_tlsauth_password, :string, 252
+      option :easy, :proxy_tlsauth_type, :enum, 253, [:none, :srp]
+      option :easy, :proxy_sslcert, :string, 254
+      option :easy, :proxy_sslcerttype, :string, 255
+      option :easy, :proxy_sslkey, :string, 256
+      option :easy, :proxy_sslkeytype, :string, 257
+      option :easy, :proxy_keypasswd, :string, 258
+      option_alias :easy, :proxy_keypasswd, :proxy_sslcertpasswd
+      option_alias :easy, :proxy_keypasswd, :proxy_sslkeypasswd
+      option :easy, :proxy_ssl_cipher_list, :string, 259
+      option :easy, :proxy_crlfile, :string, 260
+      option :easy, :proxy_ssl_options, :bitmask, 261, [nil, :allow_beast]
+      option :easy, :pre_proxy, :string, 262
+      option :easy, :proxy_pinnedpublickey, :string, 263
+      option_alias :easy, :proxy_pinnedpublickey, :proxy_pinned_public_key
+      option :easy, :proxy_issuercert, :string, 296
       ## SSH OPTIONS
       option :easy, :ssh_auth_types, :bitmask, 151, [:none, :publickey, :password, :host, :keyboard, :agent, {:any => [:all], :default => [:any]}]
       option :easy, :ssh_host_public_key_md5, :string, 162

data/lib/ethon/curls/settings.rb

@@ -2,6 +2,8 @@
 module Ethon
   module Curl
     callback :callback, [:pointer, :size_t, :size_t, :pointer], :size_t
+    callback :socket_callback, [:pointer, :int, :poll_action, :pointer, :pointer], :multi_code
+    callback :timer_callback, [:pointer, :long, :pointer], :multi_code
     callback :debug_callback, [:pointer, :debug_info_type, :pointer, :size_t, :pointer], :int
     callback :progress_callback, [:pointer, :long_long, :long_long, :long_long, :long_long], :int
     ffi_lib_flags :now, :global

data/lib/ethon/easy/informations.rb

@@ -71,7 +71,25 @@ module Ethon
 
         # Return the total number of redirections that were
         # actually followed.
-        :redirect_count => :long
+        :redirect_count => :long,
+
+        # Return the bytes, the total amount of bytes that were uploaded
+        :size_upload => :double,
+
+        # Return the bytes, the total amount of bytes that were downloaded.
+        # The amount is only for the latest transfer and will be reset again
+        # for each new transfer. This counts actual payload data, what's
+        # also commonly called body. All meta and header data are excluded
+        # and will not be counted in this number.
+        :size_download => :double,
+
+        # Return the bytes/second, the average upload speed that curl
+        # measured for the complete upload
+        :speed_upload => :double,
+
+        # Return the bytes/second, the average download speed that curl
+        # measured for the complete download
+        :speed_download => :double
       }
 
       AVAILABLE_INFORMATIONS.each do |name, type|

data/lib/ethon/easy/operations.rb

@@ -4,20 +4,6 @@ module Ethon
     # This module contains the logic to prepare and perform
     # an easy.
     module Operations
-      
-      class PointerHelper
-        class<<self
-          def synchronize( &block )
-            (@mutex ||= Mutex.new).synchronize( &block )
-          end
-
-          def release( pointer )
-            synchronize { Curl.easy_cleanup pointer }
-          end
-        end
-        synchronize{}
-      end
-
       # Returns a pointer to the curl easy handle.
       #
       # @example Return the handle.
@@ -25,7 +11,7 @@ module Ethon
       #
       # @return [ FFI::Pointer ] A pointer to the curl easy handle.
       def handle
-        @handle ||= FFI::AutoPointer.new(Curl.easy_init, PointerHelper.method(:release) )
+        @handle ||= FFI::AutoPointer.new(Curl.easy_init, Curl.method(:easy_cleanup))
       end
 
       # Sets a pointer to the curl easy handle.

data/lib/ethon/multi/operations.rb

@@ -24,12 +24,16 @@ module Ethon
       #
       # @return [ void ]
       def init_vars
-        @timeout = ::FFI::MemoryPointer.new(:long)
-        @timeval = Curl::Timeval.new
-        @fd_read = Curl::FDSet.new
-        @fd_write = Curl::FDSet.new
-        @fd_excep = Curl::FDSet.new
-        @max_fd = ::FFI::MemoryPointer.new(:int)
+        if @execution_mode == :perform
+          @timeout = ::FFI::MemoryPointer.new(:long)
+          @timeval = Curl::Timeval.new
+          @fd_read = Curl::FDSet.new
+          @fd_write = Curl::FDSet.new
+          @fd_excep = Curl::FDSet.new
+          @max_fd = ::FFI::MemoryPointer.new(:int)
+        elsif @execution_mode == :socket_action
+          @running_count_pointer = FFI::MemoryPointer.new(:int)
+        end
       end
 
       # Perform multi.
@@ -39,6 +43,8 @@ module Ethon
       # @example Perform multi.
       #   multi.perform
       def perform
+        ensure_execution_mode(:perform)
+
         Ethon.logger.debug(STARTED_MULTI)
         while ongoing?
           run
@@ -67,9 +73,38 @@ module Ethon
         )
       end
 
-      private
+      # Continue execution with an external IO loop.
+      #
+      # @example When no sockets are ready yet, or to begin.
+      #   multi.socket_action
+      #
+      # @example When a socket is readable
+      #   multi.socket_action(io_object, [:in])
+      #
+      # @example When a socket is readable and writable
+      #   multi.socket_action(io_object, [:in, :out])
+      #
+      # @return [ Symbol ] The Curl.multi_socket_action return code.
+      def socket_action(io = nil, readiness = 0)
+        ensure_execution_mode(:socket_action)
+
+        fd = if io.nil?
+          ::Ethon::Curl::SOCKET_TIMEOUT
+        elsif io.is_a?(Integer)
+          io
+        else
+          io.fileno
+        end
+
+        code = Curl.multi_socket_action(handle, fd, readiness, @running_count_pointer)
+        @running_count = @running_count_pointer.read_int
 
-      # Return wether the multi still requests or not.
+        check
+
+        code
+      end
+
+      # Return whether the multi still contains requests or not.
       #
       # @example Return if ongoing.
       #   multi.ongoing?
@@ -79,6 +114,8 @@ module Ethon
         easy_handles.size > 0 || (!defined?(@running_count) || running_count > 0)
       end
 
+      private
+
       # Get timeout.
       #
       # @example Get timeout.

data/lib/ethon/multi.rb

@@ -74,10 +74,21 @@ module Ethon
     # is 1 will not be treated as unlimited. Instead it will open only 1 
     # connection and try to pipeline on it.
     # (Added in 7.30.0)
+    # @option options :execution_mode [Boolean]
+    #  Either :perform (default) or :socket_action, specifies the usage
+    #  method that will be used on this multi object. The default :perform
+    #  mode provides a #perform function that uses curl_multi_perform
+    #  behind the scenes to automatically continue execution until all
+    #  requests have completed. The :socket_action mode provides an API
+    #  that allows the {Multi} object to be integrated into an external
+    #  IO loop, by calling #socket_action and responding to the 
+    #  socketfunction and timerfunction callbacks, using the underlying
+    #  curl_multi_socket_action semantics.
     #
     # @return [ Multi ] The new multi.
     def initialize(options = {})
       Curl.init
+      @execution_mode = options.delete(:execution_mode) || :perform
       set_attributes(options)
       init_vars
     end
@@ -100,5 +111,16 @@ module Ethon
         method("#{key}=").call(value)
       end
     end
+
+    private
+
+    # Internal function to gate functions to a specific execution mode
+    #
+    # @raise ArgumentError
+    #
+    # @api private
+    def ensure_execution_mode(expected_mode)
+      raise ArgumentError, "Expected the Multi to be in #{expected_mode} but it was in #{@execution_mode}" if expected_mode != @execution_mode
+    end
   end
 end

data/lib/ethon/version.rb

@@ -2,5 +2,5 @@
 module Ethon
 
   # Ethon version.
-  VERSION = '0.14.0'
+  VERSION = '0.15.0'
 end

data/spec/ethon/easy/informations_spec.rb

@@ -94,5 +94,27 @@ describe Ethon::Easy::Informations do
     end
   end
 
+  describe "#size_upload" do
+    it "returns float" do
+      expect(easy.size_upload).to be_a(Float)
+    end
+  end
+
+  describe "#size_download" do
+    it "returns float" do
+      expect(easy.size_download).to be_a(Float)
+    end
+  end
 
+  describe "#speed_upload" do
+    it "returns float" do
+      expect(easy.speed_upload).to be_a(Float)
+    end
+  end
+
+  describe "#speed_download" do
+    it "returns float" do
+      expect(easy.speed_download).to be_a(Float)
+    end
+  end
 end

data/spec/ethon/easy/mirror_spec.rb

@@ -10,6 +10,7 @@ describe Ethon::Easy::Mirror do
       :return_code, :response_code, :response_body, :response_headers,
       :total_time, :starttransfer_time, :appconnect_time,
       :pretransfer_time, :connect_time, :namelookup_time, :redirect_time,
+      :size_upload, :size_download, :speed_upload, :speed_upload,
       :effective_url, :primary_ip, :redirect_count, :debug_info
     ].each do |name|
       it "contains #{name}" do

data/spec/ethon/multi/options_spec.rb

@@ -20,6 +20,119 @@ describe Ethon::Multi::Options do
     end
   end
 
+  context "socket_action mode" do
+    let(:multi) { Ethon::Multi.new(execution_mode: :socket_action) }
+
+    describe "#socketfunction callbacks" do
+      it "allows multi_code return values" do
+        calls = []
+        multi.socketfunction = proc do |handle, sock, what, userp, socketp|
+          calls << what
+          :ok
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        expect(calls).to eq([])
+        5.times do
+          multi.socket_action
+          break unless calls.empty?
+          sleep 0.1
+        end
+        expect(calls.last).to eq(:in).or(eq(:out))
+        multi.delete(easy)
+        expect(calls.last).to eq(:remove)
+      end
+
+      it "allows integer return values (compatibility)" do
+        called = false
+        multi.socketfunction = proc do |handle, sock, what, userp, socketp|
+          called = true
+          0
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        5.times do
+          multi.socket_action
+          break if called
+          sleep 0.1
+        end
+        multi.delete(easy)
+
+        expect(called).to be_truthy
+      end
+
+      it "errors on invalid return codes" do
+        called = false
+        multi.socketfunction = proc do |handle, sock, what, userp, socketp|
+          called = true
+          "hi"
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        expect {
+          5.times do
+            multi.socket_action
+            break if called
+            sleep 0.1
+          end
+        }.to raise_error(ArgumentError)
+        expect { multi.delete(easy) }.to raise_error(ArgumentError)
+      end
+    end
+
+    describe "#timerfunction callbacks" do
+      it "allows multi_code return values" do
+        calls = []
+        multi.timerfunction = proc do |handle, timeout_ms, userp|
+          calls << timeout_ms
+          :ok
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        expect(calls.last).to be >= 0 # adds an immediate timeout
+
+        multi.delete(easy)
+        expect(calls.last).to eq(-1) # cancels the timer
+      end
+
+      it "allows integer return values (compatibility)" do
+        called = false
+        multi.timerfunction = proc do |handle, timeout_ms, userp|
+          called = true
+          0
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        multi.socket_action
+        multi.delete(easy)
+
+        expect(called).to be_truthy
+      end
+
+      it "errors on invalid return codes" do
+        called = false
+        multi.timerfunction = proc do |handle, timeout_ms, userp|
+          called = true
+          "hi"
+        end
+
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        expect { multi.add(easy) }.to raise_error(ArgumentError)
+      end
+    end
+  end
+
   describe "#value_for" do
     context "when option in bool" do
       context "when value true" do

data/spec/ethon/multi_spec.rb

@@ -8,6 +8,16 @@ describe Ethon::Multi do
       Ethon::Multi.new
     end
 
+    context "with default options" do
+      it "allows running #perform with the default execution_mode" do
+        Ethon::Multi.new.perform
+      end
+
+      it "refuses to run #socket_action" do
+        expect { Ethon::Multi.new.socket_action }.to raise_error(ArgumentError)
+      end
+    end
+
     context "when options not empty" do
       context "when pipelining is set" do
         let(:options) { { :pipelining => true } }
@@ -17,6 +27,126 @@ describe Ethon::Multi do
           Ethon::Multi.new(options)
         end
       end
+
+      context "when execution_mode option is :socket_action" do
+        let(:options) { { :execution_mode => :socket_action } }
+        let(:multi) { Ethon::Multi.new(options) }
+
+        it "refuses to run #perform" do
+          expect { multi.perform }.to raise_error(ArgumentError)
+        end
+
+        it "allows running #socket_action" do
+          multi.socket_action
+        end
+      end
+    end
+  end
+
+  describe "#socket_action" do
+    let(:options) { { :execution_mode => :socket_action } }
+    let(:select_state) { { :readers => [], :writers => [], :timeout => 0 } }
+    let(:multi) {
+      multi = Ethon::Multi.new(options)
+      multi.timerfunction = proc do |handle, timeout_ms, userp|
+        timeout_ms = nil if timeout_ms == -1
+        select_state[:timeout] = timeout_ms
+        :ok
+      end
+      multi.socketfunction = proc do |handle, sock, what, userp, socketp|
+        case what
+        when :remove
+          select_state[:readers].delete(sock)
+          select_state[:writers].delete(sock)
+        when :in
+          select_state[:readers].push(sock) unless select_state[:readers].include? sock
+          select_state[:writers].delete(sock)
+        when :out
+          select_state[:readers].delete(sock)
+          select_state[:writers].push(sock) unless select_state[:writers].include? sock
+        when :inout
+          select_state[:readers].push(sock) unless select_state[:readers].include? sock
+          select_state[:writers].push(sock) unless select_state[:writers].include? sock
+        else
+          raise ArgumentError, "invalid value for 'what' in socketfunction callback"
+        end
+        :ok
+      end
+      multi
+    }
+
+    def fds_to_ios(fds)
+      fds.map do |fd|
+        IO.for_fd(fd).tap { |io| io.autoclose = false }
+      end
+    end
+
+    def perform_socket_action_until_complete
+      multi.socket_action # start things off
+
+      while multi.ongoing?
+        readers, writers, _ = IO.select(
+          fds_to_ios(select_state[:readers]),
+          fds_to_ios(select_state[:writers]),
+          [],
+          select_state[:timeout]
+        )
+
+        to_notify = Hash.new { |hash, key| hash[key] = [] }
+        unless readers.nil?
+          readers.each do |reader|
+            to_notify[reader] << :in
+          end
+        end
+        unless writers.nil?
+          writers.each do |writer|
+            to_notify[writer] << :out
+          end
+        end
+
+        to_notify.each do |io, readiness|
+          multi.socket_action(io, readiness)
+        end
+
+        # if we didn't have anything to notify, then we timed out
+        multi.socket_action if to_notify.empty?
+      end
+    ensure
+      multi.easy_handles.dup.each do |h|
+        multi.delete(h)
+      end
+    end
+
+    it "supports an end-to-end request" do
+      easy = Ethon::Easy.new
+      easy.url = "http://localhost:3001/"
+      multi.add(easy)
+
+      perform_socket_action_until_complete
+
+      expect(multi.ongoing?).to eq(false)
+    end
+
+    it "supports multiple concurrent requests" do
+      handles = []
+      10.times do
+        easy = Ethon::Easy.new
+        easy.url = "http://localhost:3001/?delay=1"
+        multi.add(easy)
+        handles << easy
+      end
+
+      start = Time.now
+      perform_socket_action_until_complete
+      duration = Time.now - start
+      
+      # these should have happened concurrently
+      expect(duration).to be < 2
+      expect(multi.ongoing?).to eq(false)
+
+      handles.each do |handle|
+        expect(handle.response_code).to eq(200)
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ethon
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Hans Hasselberg
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-26 00:00:00.000000000 Z
+date: 2021-10-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ffi