ethon 0.18.0 → 0.19.1

data/lib/ethon/multi/options.rb

@@ -9,25 +9,89 @@ module Ethon
       # Sets max_total_connections option.
       #
       # @example Set max_total_connections option.
-      #   multi.max_total_conections = $value
+      #   multi.max_total_connections = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ Integer ] value The value to set.
       #
       # @return [ void ]
       def max_total_connections=(value)
-        Curl.set_option(:max_total_connections, value_for(value, :int), handle, :multi)
+        Curl.set_option(:max_total_connections, value_for(value, :int, :max_total_connections), handle, :multi)
+      end
+
+      # Sets max_concurrent_streams option.
+      #
+      # @example Set max_concurrent_streams option.
+      #   multi.max_concurrent_streams = $value
+      #
+      # @param [ Integer ] value The value to set.
+      #
+      # @return [ void ]
+      def max_concurrent_streams=(value)
+        Curl.set_option(:max_concurrent_streams, value_for(value, :int, :max_concurrent_streams), handle, :multi)
       end
 
       # Sets maxconnects option.
       #
+      # An explicit value also disables the automatic sizing described in
+      # #raise_default_maxconnects.
+      #
       # @example Set maxconnects option.
       #   multi.maxconnects = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ Integer ] value The value to set.
       #
       # @return [ void ]
       def maxconnects=(value)
-        Curl.set_option(:maxconnects, value_for(value, :int), handle, :multi)
+        @maxconnects_configured = true unless value.nil?
+        set_maxconnects(value)
+      end
+
+      # Sets max_host_connections option.
+      #
+      # @example Set max_host_connections option.
+      #   multi.max_host_connections = $value
+      #
+      # @param [ Integer ] value The value to set.
+      #
+      # @return [ void ]
+      def max_host_connections=(value)
+        Curl.set_option(:max_host_connections, value_for(value, :int, :max_host_connections), handle, :multi)
+      end
+
+      # Sets max_pipeline_length option.
+      #
+      # @example Set max_pipeline_length option.
+      #   multi.max_pipeline_length = $value
+      #
+      # @param [ Integer ] value The value to set.
+      #
+      # @return [ void ]
+      def max_pipeline_length=(value)
+        Curl.set_option(:max_pipeline_length, value_for(value, :int, :max_pipeline_length), handle, :multi)
+      end
+
+      # Sets content_length_penalty_size option.
+      #
+      # @example Set content_length_penalty_size option.
+      #   multi.content_length_penalty_size = $value
+      #
+      # @param [ Integer ] value The value to set.
+      #
+      # @return [ void ]
+      def content_length_penalty_size=(value)
+        Curl.set_option(:content_length_penalty_size, value, handle, :multi)
+      end
+
+      # Sets chunk_length_penalty_size option.
+      #
+      # @example Set chunk_length_penalty_size option.
+      #   multi.chunk_length_penalty_size = $value
+      #
+      # @param [ Integer ] value The value to set.
+      #
+      # @return [ void ]
+      def chunk_length_penalty_size=(value)
+        Curl.set_option(:chunk_length_penalty_size, value, handle, :multi)
       end
 
       # Sets pipelining option.
@@ -35,11 +99,11 @@ module Ethon
       # @example Set pipelining option.
       #   multi.pipelining = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ Integer ] value The value to set.
       #
       # @return [ void ]
       def pipelining=(value)
-        Curl.set_option(:pipelining, value_for(value, :int), handle, :multi)
+        Curl.set_option(:pipelining, value_for(value, :int, :pipelining), handle, :multi)
       end
 
       # Sets socketdata option.
@@ -47,7 +111,7 @@ module Ethon
       # @example Set socketdata option.
       #   multi.socketdata = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ FFI::Pointer, nil ] value The value to set.
       #
       # @return [ void ]
       def socketdata=(value)
@@ -59,11 +123,13 @@ module Ethon
       # @example Set socketfunction option.
       #   multi.socketfunction = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ Proc, nil ] value The value to set.
       #
       # @return [ void ]
       def socketfunction=(value)
         Curl.set_option(:socketfunction, value_for(value, :string), handle, :multi)
+        retain_callback(:socketfunction, value)
+        value
       end
 
       # Sets timerdata option.
@@ -71,7 +137,7 @@ module Ethon
       # @example Set timerdata option.
       #   multi.timerdata = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ FFI::Pointer, nil ] value The value to set.
       #
       # @return [ void ]
       def timerdata=(value)
@@ -83,15 +149,50 @@ module Ethon
       # @example Set timerfunction option.
       #   multi.timerfunction = $value
       #
-      # @param [ String ] value The value to set.
+      # @param [ Proc, nil ] value The value to set.
       #
       # @return [ void ]
       def timerfunction=(value)
         Curl.set_option(:timerfunction, value_for(value, :string), handle, :multi)
+        retain_callback(:timerfunction, value)
+        value
       end
 
       private
 
+      # Raise the default connection cache size to twice the largest number
+      # of easy handles ever attached at once.
+      #
+      # libcurl derives its default from the number of handles attached when
+      # a connection is returned. Ethon detaches each handle as soon as its
+      # transfer finishes, so that number falls during a run and libcurl
+      # closes connections that later requests could have reused.
+      #
+      # @param [ Integer ] count The number of attached handles after the
+      #   pending add.
+      #
+      # @return [ void ]
+      def raise_default_maxconnects(count)
+        return if @maxconnects_configured
+        return if count <= (@maxconnects_high_water ||= 0)
+
+        @maxconnects_high_water = count
+        set_maxconnects(2 * count)
+      end
+
+      def set_maxconnects(value)
+        Curl.set_option(:maxconnects, value_for(value, :int, :maxconnects), handle, :multi)
+      end
+
+      def retain_callback(option, callback)
+        if callback.nil?
+          @procs.delete(option) if @procs
+        else
+          (@procs ||= {})[option] = callback
+        end
+        Curl::MultiHandle.retain_callback(handle, option, callback)
+      end
+
       # Return the value to set to multi handle. It is converted with the help
       # of bool_options, enum_options and int_options.
       #
@@ -105,13 +206,14 @@ module Ethon
         if type == :bool
           value ? 1 : 0
         elsif type == :int
-          value.to_i
+          Ethon::Curls::Options.integer_value(option, value)
         elsif value.is_a?(String)
           Ethon::Easy::Util.escape_zero_byte(value)
         else
           value
         end
       end
+
     end
   end
 end

data/lib/ethon/multi/stack.rb

@@ -5,14 +5,18 @@ module Ethon
     # This module provides the multi stack behaviour.
     module Stack
 
-      # Return easy handles.
+      # Return the easy handles currently added to this multi.
+      #
+      # The returned array is a snapshot. Adding or deleting entries there does
+      # not change the multi, because libcurl would still reference the handles.
+      # Use #add and #delete instead.
       #
       # @example Return easy handles.
       #   multi.easy_handles
       #
-      # @return [ Array ] The easy handles.
+      # @return [ Array<Easy> ] The easy handles.
       def easy_handles
-        @easy_handles ||= []
+        stack.dup.freeze
       end
 
       # Add an easy to the stack.
@@ -22,13 +26,70 @@ module Ethon
       #
       # @param [ Easy ] easy The easy to add.
       #
+      # A failed add leaves the Easy untouched: its previous response and any
+      # streamed response file stay as they were.
+      #
+      # @raise [ TypeError ] If easy is not an Ethon::Easy.
       # @raise [ Ethon::Errors::MultiAdd ] If adding an easy failed.
       def add(easy)
-        return nil if easy_handles.include?(easy)
+        return nil if stack.include?(easy)
+        unless easy.is_a?(Ethon::Easy)
+          raise TypeError, "expected an Ethon::Easy, got #{easy.class}"
+        end
+
+        # Another Multi may still own the handle natively. Reject before
+        # resetting its transfer state or touching its response stream.
+        if easy.instance_variable_get(:@multi)
+          raise Errors::EthonError, "Easy is still attached to a Multi; remove it first"
+        end
 
-        code = Curl.multi_add_handle(handle, easy.handle)
-        raise Errors::MultiAdd.new(code, easy) unless code == :ok
-        easy_handles << easy
+        # Resolve every handle before touching the easy so a cleaned up or
+        # forked Multi or Easy cannot be changed while raising.
+        multi_handle = handle
+        easy_handle = easy.handle
+
+        # Pin libcurl's default connection cache size before the handle is
+        # attached. Finished handles are detached immediately, which would
+        # otherwise make the default shrink during a run and close reusable
+        # connections.
+        raise_default_maxconnects(stack.size + 1)
+
+        stack << easy
+        begin
+          code = Curl.multi_add_handle(multi_handle, easy_handle)
+        rescue Exception
+          # The native add may have completed before a socket or timer callback
+          # raised, so make sure libcurl does not keep an untracked handle.
+          detach_added_handle(easy, multi_handle, easy_handle)
+          raise
+        end
+
+        unless code == :ok
+          stack.delete(easy)
+          raise Errors::MultiAdd.new(code, easy)
+        end
+
+        # Cleanup must not free an easy that is still attached to this multi.
+        easy.instance_variable_set(:@multi, self)
+
+        # Only a successful add starts a new transfer. Resetting the previous
+        # response and opening its output stream before libcurl accepted the
+        # handle would make a failed add discard the response body and truncate
+        # a streamed response file.
+        begin
+          # A completion callback can add the Easy to this Multi before the
+          # previous transfer's destination is finished. Close or flush it
+          # first so prepare opens a fresh destination for this transfer.
+          easy.finish_response_body_io
+          easy.prepare_response_body_io
+          easy.reset_transfer_state
+          easy.clear_callback_error
+        rescue Exception
+          rollback_add(easy, multi_handle, easy_handle)
+          raise
+        end
+
+        easy_handles
       end
 
       # Delete an easy from stack.
@@ -39,11 +100,121 @@ module Ethon
       #
       # @raise [ Ethon::Errors::MultiRemove ] If removing an easy failed.
       def delete(easy)
-        if easy_handles.delete(easy)
-          code = Curl.multi_remove_handle(handle, easy.handle)
-          raise Errors::MultiRemove.new(code, handle) unless code == :ok
+        if (index = stack.index(easy))
+          easy_handle = nil
+          stack.delete_at(index)
+          begin
+            easy_handle = easy.handle
+            code = Curl.multi_remove_handle(handle, easy_handle)
+          rescue Exception
+            # A socket callback can raise after libcurl has already detached
+            # the handle. Keep the stack in sync with the native multi instead
+            # of tracking a handle libcurl no longer owns.
+            if native_detached?(easy_handle)
+              begin
+                finish_removed_easy(easy)
+              rescue StandardError
+                # Keep the original remove error.
+              end
+            else
+              stack.insert(index, easy)
+            end
+            raise
+          end
+          unless code == :ok
+            stack.insert(index, easy)
+            raise Errors::MultiRemove.new(code, easy)
+          end
+
+          finish_removed_easy(easy)
         end
       end
+
+      private
+
+      # The mutable stack. Only this Multi may change it; every native change
+      # has to go through #add and #delete.
+      def stack
+        @easy_handles ||= []
+      end
+
+      # Finish an Easy that is no longer attached to this Multi.
+      def finish_removed_easy(easy)
+        if easy.instance_variable_get(:@multi).equal?(self)
+          easy.instance_variable_set(:@multi, nil)
+        end
+
+        begin
+          easy.finish_response_body_io
+        ensure
+          @running_count = nil if stack.empty?
+        end
+      end
+
+      # Whether libcurl detached the handle before a callback raised. Older
+      # libcurl cannot answer, so assume it is still attached and keep it
+      # tracked.
+      def native_detached?(easy_handle)
+        return false unless easy_handle
+        return false unless Curl.respond_to?(:multi_get_handles)
+
+        !native_handle?(easy_handle)
+      rescue StandardError
+        false
+      end
+
+      # Undo a native add whose Easy could not be prepared for the transfer.
+      def rollback_add(easy, multi_handle, easy_handle)
+        begin
+          easy.finish_response_body_io
+        rescue StandardError
+          # Best effort; the add is already being rolled back.
+        end
+
+        detach_added_handle(easy, multi_handle, easy_handle)
+      end
+
+      # Detach a handle that libcurl may have attached before an exception
+      # interrupted the add. The easy stays tracked when it may still be
+      # attached so that Multi#cleanup can retry instead of leaving libcurl
+      # with an untracked attached handle.
+      def detach_added_handle(easy, multi_handle, easy_handle)
+        result = begin
+          Curl.multi_remove_handle(multi_handle, easy_handle)
+        rescue Exception
+          nil
+        end
+
+        attached = case result
+                   when :ok, :bad_easy_handle
+                     false
+                   when nil
+                     # libcurl detaches the easy before it invokes a socket or
+                     # timer callback, so a callback exception is not enough to
+                     # assume it is still attached.
+                     native_handle?(easy_handle)
+                   else
+                     true
+                   end
+
+        if attached
+          easy.instance_variable_set(:@multi, self)
+        else
+          stack.delete(easy)
+          easy.instance_variable_set(:@multi, nil)
+        end
+      end
+
+      # Whether libcurl still knows the native easy handle. libcurl can only
+      # answer that question from 7.88.0 on; older versions assume the handle
+      # was detached, which matches the order inside Multi#delete.
+      def native_handle?(easy_handle)
+        return false unless Curl.respond_to?(:multi_get_handles)
+
+        native_handles.any? { |handle| handle == easy_handle }
+      rescue NotImplementedError
+        false
+      end
     end
   end
 end

data/lib/ethon/multi.rb

@@ -12,6 +12,8 @@ module Ethon
     include Ethon::Multi::Operations
     include Ethon::Multi::Options
 
+    EXECUTION_MODES = [:perform, :socket_action].freeze
+
     # Create a new multi. Initialize curl in case
     # it didn't happen before.
     #
@@ -20,18 +22,19 @@ module Ethon
     #
     # @param [ Hash ] options The options.
     #
-    # @option options :socketdata [String]
+    # @option options :socketdata [FFI::Pointer]
     #  Pass a pointer to whatever you want passed to the
     #  curl_socket_callback's forth argument, the userp pointer. This is not
     #  used by libcurl but only passed-thru as-is. Set the callback pointer
-    #  with CURLMOPT_SOCKETFUNCTION.
-    # @option options :pipelining [Boolean]
-    #  Pass a long set to 1 to enable or 0 to disable. Enabling pipelining
-    #  on a multi handle will make it attempt to perform HTTP Pipelining as
-    #  far as possible for transfers using this handle. This means that if
-    #  you add a second request that can use an already existing connection,
-    #  the second request will be "piped" on the same connection rather than
-    #  being executed in parallel. (Added in 7.16.0)
+    #  with CURLMOPT_SOCKETFUNCTION. Keep the pointer alive for as long as the
+    #  Multi may invoke the callback.
+    # @option options :pipelining [Integer]
+    #  Pass a bitmask: 0 for CURLPIPE_NOTHING, 1 for CURLPIPE_HTTP1, or 2 for
+    #  CURLPIPE_MULTIPLEX. CURLPIPE_HTTP1 is deprecated and has no effect
+    #  since libcurl 7.62.0. CURLPIPE_MULTIPLEX allows libcurl to multiplex a
+    #  new transfer over an existing HTTP/2 or HTTP/3 connection when
+    #  possible, and is the default since libcurl 7.62.0. (Added in 7.16.0;
+    #  multiplex support added in 7.43.0.)
     # @option options :timerfunction [Proc]
     #  Pass a pointer to a function matching the curl_multi_timer_callback
     #  prototype. This function will then be called when the timeout value
@@ -44,16 +47,21 @@ module Ethon
     #  only when the fixed future timeout time actually changes. See also
     #  CURLMOPT_TIMERDATA. This callback can be used instead of, or in
     #  addition to, curl_multi_timeout(3). (Added in 7.16.0)
-    # @option options :timerdata [String]
+    # @option options :timerdata [FFI::Pointer]
     #  Pass a pointer to whatever you want passed to the
     #  curl_multi_timer_callback's third argument, the userp pointer. This
     #  is not used by libcurl but only passed-thru as-is. Set the callback
-    #  pointer with CURLMOPT_TIMERFUNCTION. (Added in 7.16.0)
+    #  pointer with CURLMOPT_TIMERFUNCTION. Keep the pointer alive for as long
+    #  as the Multi may invoke the callback. (Added in 7.16.0)
     # @option options :maxconnects [Integer]
     #  Pass a long. The set number will be used as the maximum amount of
-    #  simultaneously open connections that libcurl may cache. Default is
-    #  10, and libcurl will enlarge the size for each added easy handle to
-    #  make it fit 4 times the number of added easy handles.
+    #  simultaneously open connections that libcurl may cache. When this is
+    #  not set, Ethon sizes the cache to twice the largest number of easy
+    #  handles attached to this Multi at once. libcurl derives its own default
+    #  from the handles attached when a connection is returned, but Ethon
+    #  detaches finished handles, which would otherwise shrink the cache during
+    #  a run. Pass an explicit value to cap the cache yourself, or 0 to use
+    #  libcurl's dynamic default.
     #  By setting this option, you can prevent the cache size from growing
     #  beyond the limit set by you.
     #  When the cache is full, curl closes the oldest one in the cache to
@@ -61,6 +69,23 @@ module Ethon
     #  This option is for the multi handle's use only, when using the easy
     #  interface you should instead use the CURLOPT_MAXCONNECTS option.
     #  (Added in 7.16.3)
+    # @option options :max_host_connections [Integer]
+    # Pass a long. The set number will be used as the maximum amount of
+    # simultaneously open connections to a single host.
+    # (Added in 7.30.0)
+    # @option options :max_pipeline_length [Integer]
+    # Pass a long. The set number will be used as the maximum amount of
+    # requests in a pipeline. This option is obsolete in libcurl 7.62.0 and
+    # later, where HTTP/1 pipelining has no effect.
+    # (Added in 7.16.0)
+    # @option options :content_length_penalty_size [Integer]
+    # Pass a curl_off_t. This option sets the size of the content length
+    # penalty in bytes. It is used by the pipelining code.
+    # (Added in 7.16.0)
+    # @option options :chunk_length_penalty_size [Integer]
+    # Pass a curl_off_t. This option sets the size of the chunk length penalty
+    # in bytes. It is used by the pipelining code.
+    # (Added in 7.16.0)
     # @option options :max_total_connections [Integer]
     # Pass a long. The set number will be used as the maximum amount of 
     # simultaneously open connections in total. For each new session, 
@@ -74,7 +99,7 @@ module Ethon
     # is 1 will not be treated as unlimited. Instead it will open only 1 
     # connection and try to pipeline on it.
     # (Added in 7.30.0)
-    # @option options :execution_mode [Boolean]
+    # @option options :execution_mode [Symbol]
     #  Either :perform (default) or :socket_action, specifies the usage
     #  method that will be used on this multi object. The default :perform
     #  mode provides a #perform function that uses curl_multi_perform
@@ -88,7 +113,11 @@ module Ethon
     # @return [ Multi ] The new multi.
     def initialize(options = {})
       Curl.init
+      options = options.dup
       @execution_mode = options.delete(:execution_mode) || :perform
+      unless EXECUTION_MODES.include?(@execution_mode)
+        raise ArgumentError, "Unknown execution mode #{@execution_mode.inspect}"
+      end
       set_attributes(options)
       init_vars
     end
@@ -114,6 +143,13 @@ module Ethon
 
     private
 
+    # Ruby cannot duplicate the native multi handle or its Ruby-side
+    # bookkeeping safely. A shallow copy would share the easy-handle stack and
+    # the native handle, so cleaning up one copy would invalidate the other.
+    def initialize_copy(_other)
+      raise TypeError, "cannot duplicate an Ethon::Multi"
+    end
+
     # Internal function to gate functions to a specific execution mode
     #
     # @raise ArgumentError

data/lib/ethon/version.rb

@@ -2,5 +2,5 @@
 module Ethon
 
   # Ethon version.
-  VERSION = '0.18.0'
+  VERSION = '0.19.1'
 end

data/lib/ethon.rb

@@ -2,23 +2,22 @@
 require 'logger'
 require 'ffi'
 require 'thread'
-begin
-  require 'mime/types/columnar'
-rescue LoadError
-  begin
-    require 'mime/types'
-  rescue LoadError
-  end
-end
 require 'tempfile'
 
 require 'ethon/libc'
 require 'ethon/curl'
+require 'ethon/curls/mime'
 require 'ethon/easy'
+require 'ethon/easy/url'
+require 'ethon/easy/mime'
 require 'ethon/errors'
+require 'ethon/curls/introspection'
+require 'ethon/easy/headers_api'
 require 'ethon/loggable'
 require 'ethon/multi'
 require 'ethon/version'
+require 'ethon/curls/share'
+require 'ethon/easy/share'
 
 # Ethon is a very simple libcurl.
 # It provides direct access to libcurl functionality
@@ -31,6 +30,6 @@ require 'ethon/version'
 #
 # @note Please update to the latest libcurl version in order
 #   to benefit from all features and bugfixes.
-#   http://curl.haxx.se/download.html
+#   https://curl.se/download.html
 module Ethon
 end