ethon 0.18.0 → 0.19.1

data/lib/ethon/errors/multi_poll.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when curl_multi_poll fails.
+    class MultiPoll < EthonError
+      def initialize(code)
+        super("An error occurred polling the multi: #{code}")
+      end
+    end
+  end
+end

data/lib/ethon/errors/multi_remove.rb

@@ -5,7 +5,7 @@ module Ethon
     # Raises when multi_remove_handle failed.
     class MultiRemove < EthonError
       def initialize(code, easy)
-        super("An error occured removing the easy handle: #{easy} from the multi: #{code}")
+        super("An error occurred removing the easy handle: #{easy} from the multi: #{code}")
       end
     end
   end

data/lib/ethon/errors/multi_timeout.rb

@@ -5,8 +5,8 @@ module Ethon
     # Raised when multi_timeout failed.
     class MultiTimeout < EthonError
       def initialize(code)
-        super("An error occured getting the timeout: #{code}")
-        # "An error occured getting the timeout: #{code}: #{Curl.multi_strerror(code)}"
+        super("An error occurred getting the timeout: #{code}")
+        # "An error occurred getting the timeout: #{code}: #{Curl.multi_strerror(code)}"
       end
     end
   end

data/lib/ethon/errors/multi_wait.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when curl_multi_wait fails.
+    class MultiWait < EthonError
+      def initialize(code)
+        super("An error occurred waiting on the multi: #{code}")
+      end
+    end
+  end
+end

data/lib/ethon/errors/multi_wakeup.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when curl_multi_wakeup fails.
+    class MultiWakeup < EthonError
+      def initialize(code)
+        super("An error occurred waking up the multi: #{code}")
+      end
+    end
+  end
+end

data/lib/ethon/errors/select.rb

@@ -5,7 +5,7 @@ module Ethon
     # Raised when select failed.
     class Select < EthonError
       def initialize(errno)
-        super("An error occured on select: #{errno}")
+        super("An error occurred on select: #{errno}")
       end
     end
   end

data/lib/ethon/errors/set_option.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when libcurl rejects an option value.
+    class SetOption < EthonError
+      def initialize(option, code, type = :easy)
+        message = begin
+          strerror(type, code)
+        rescue StandardError
+          code.to_s
+        end
+        # Inspect so control characters in an untrusted option name cannot
+        # inject lines into logs that record the message.
+        super("libcurl failed to set the option: #{option.inspect}: #{message} (#{code})")
+      end
+
+      private
+
+      # Multi and easy codes overlap, so decode with the matching strerror.
+      def strerror(type, code)
+        func = type == :multi ? :multi_strerror : :easy_strerror
+        Ethon::Curl.respond_to?(func) ? Ethon::Curl.public_send(func, code) : code.to_s
+      end
+    end
+  end
+end

data/lib/ethon/errors/share.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when a curl share API call fails.
+    class Share < EthonError
+      def initialize(code)
+        message = if Ethon::Curl.respond_to?(:share_strerror)
+          Ethon::Curl.share_strerror(code)
+        else
+          code.to_s
+        end
+        super("An error occurred handling the share: #{message}")
+      end
+    end
+  end
+end

data/lib/ethon/errors/unsupported_option.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when libcurl does not recognize an option.
+    #
+    # This happens when Ethon knows an option that the loaded libcurl is too
+    # old to support, for example an option added in a newer libcurl release.
+    class UnsupportedOption < EthonError
+      def initialize(option)
+        version = begin
+          Ethon::Curl.version_info[:version]
+        rescue StandardError
+          nil
+        end
+        version = Ethon::Curl.version if version.nil?
+        # Inspect so control characters in an untrusted option name cannot
+        # inject lines into logs that record the message.
+        super("libcurl #{version} does not support the option: #{option.inspect}")
+      end
+    end
+  end
+end

data/lib/ethon/errors/url.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+module Ethon
+  module Errors
+
+    # Raised when a curl URL API call fails.
+    class Url < EthonError
+      def initialize(code)
+        message = if Ethon::Curl.respond_to?(:url_strerror)
+          Ethon::Curl.url_strerror(code)
+        else
+          code.to_s
+        end
+        super("An error occurred handling the URL: #{message}")
+      end
+    end
+  end
+end

data/lib/ethon/errors.rb

@@ -1,13 +1,26 @@
 # frozen_string_literal: true
 require 'ethon/errors/ethon_error'
+require 'ethon/errors/url'
 require 'ethon/errors/global_init'
+require 'ethon/errors/form_add'
 require 'ethon/errors/multi_timeout'
+require 'ethon/errors/multi_assign'
 require 'ethon/errors/multi_fdset'
+require 'ethon/errors/share'
 require 'ethon/errors/multi_add'
 require 'ethon/errors/multi_remove'
+require 'ethon/errors/header'
+require 'ethon/errors/multi_perform'
 require 'ethon/errors/select'
 require 'ethon/errors/invalid_option'
 require 'ethon/errors/invalid_value'
+require 'ethon/errors/set_option'
+require 'ethon/errors/unsupported_option'
+require 'ethon/errors/mime'
+require 'ethon/errors/fork_safety'
+require 'ethon/errors/multi_wait'
+require 'ethon/errors/multi_poll'
+require 'ethon/errors/multi_wakeup'
 
 module Ethon
 

data/lib/ethon/libc.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
+require 'etc'
+
 module Ethon
 
   # FFI Wrapper module for Libc.
@@ -13,8 +15,35 @@ module Ethon
       Gem.win_platform?
     end
 
+    def self.open_max
+      value = Etc.sysconf(Etc::SC_OPEN_MAX)
+      return value if value && value > 0
+
+      Process.getrlimit(:NOFILE).first
+    rescue NameError, NotImplementedError
+      Process.getrlimit(:NOFILE).first
+    end
+
+    # Size of the native fd_set, rounded up to whole longs. An unaligned
+    # ETHON_FD_SIZE would make select write past the Ruby-side struct.
+    def self.fd_size(value = ENV['ETHON_FD_SIZE'])
+      value = value.to_s
+      size = if value.empty?
+        [open_max, 65_536].min
+      else
+        begin
+          Integer(value, 10)
+        rescue ArgumentError, TypeError
+          raise ArgumentError, "ETHON_FD_SIZE must be a positive integer"
+        end
+      end
+      raise ArgumentError, "ETHON_FD_SIZE must be a positive integer" unless size > 0
+
+      word = FFI::Type::LONG.size
+      ((size + word - 1) / word) * word
+    end
+
     unless windows?
-      attach_function :getdtablesize, [], :int
       attach_function :free, [:pointer], :void
     end
   end

data/lib/ethon/loggable.rb

@@ -14,7 +14,7 @@ module Ethon
     #
     # @return [ Logger ] The logger.
     def logger
-      return @logger if defined?(@logger)
+      return @logger if defined?(@logger) && @logger
       @logger = rails_logger || default_logger
     end
 

data/lib/ethon/multi/operations.rb

@@ -14,7 +14,68 @@ module Ethon
       #
       # @return [ FFI::Pointer ] The multi handle.
       def handle
-        @handle ||= FFI::AutoPointer.new(Curl.multi_init, Curl.method(:multi_cleanup))
+        ensure_not_cleaned_up!
+
+        handle = @handle ||= begin
+          # Enforce fork safety before creating a handle for a Multi that was
+          # constructed before the fork but never touched.
+          Curl.check_pid!
+          pointer = Curl.multi_init
+          raise NoMemoryError, "curl_multi_init failed" if pointer.null?
+          Curl::MultiHandle.new(pointer)
+        end
+        handle.check_pid!
+        handle
+      end
+
+      # Clean up the multi.
+      #
+      # Removes all attached easy handles before freeing the native multi
+      # handle. Cleanup is terminal: the Multi cannot be used afterwards.
+      #
+      # @return [ nil ]
+      def cleanup
+        ensure_not_cleaned_up!
+        ensure_not_executing!
+        # A handle created before a fork must not be released in the child.
+        @handle.check_pid! if @handle
+
+        error = nil
+        begin
+          stack.dup.each do |easy|
+            begin
+              delete(easy)
+            rescue StandardError => e
+              # Keep detaching the other handles. A failed close or native
+              # detach must not strand every remaining handle on a Multi whose
+              # native handle is about to be freed.
+              error ||= e
+            end
+          end
+        ensure
+          # Cleanup is terminal. Release the native handle even when detaching
+          # one of the easy handles fails. Handles that could not be detached
+          # keep their multi ownership so they cannot be reused unsafely.
+          begin
+            @handle.free if @handle
+          ensure
+            stack.each do |easy|
+              begin
+                easy.finish_response_body_io
+              rescue StandardError
+                # Cleanup is best effort; keep tearing down the multi even if
+                # closing a streamed response fails.
+              end
+            end
+            stack.clear
+            @handle = nil
+            @running_count = nil
+            @assigned_sockets = nil
+            @procs = nil
+            @cleaned_up = true
+          end
+        end
+        raise error if error
       end
 
       # Initialize variables.
@@ -26,13 +87,26 @@ module Ethon
       def init_vars
         if @execution_mode == :perform
           @timeout = ::FFI::MemoryPointer.new(:long)
-          @timeval = Curl::Timeval.new
-          @fd_read = Curl::FDSet.new
-          @fd_write = Curl::FDSet.new
-          @fd_excep = Curl::FDSet.new
-          @max_fd = ::FFI::MemoryPointer.new(:int)
+          @running_count_pointer = ::FFI::MemoryPointer.new(:int)
+          @msgs_left = ::FFI::MemoryPointer.new(:int)
+          @wait_strategy = if Curl.respond_to?(:multi_poll)
+            :poll
+          elsif Curl.respond_to?(:multi_wait)
+            :wait
+          else
+            :select
+          end
+
+          if @wait_strategy == :select
+            @timeval = Curl::Timeval.new
+            @fd_read = Curl::FDSet.new
+            @fd_write = Curl::FDSet.new
+            @fd_excep = Curl::FDSet.new
+            @max_fd = ::FFI::MemoryPointer.new(:int)
+          end
         elsif @execution_mode == :socket_action
           @running_count_pointer = FFI::MemoryPointer.new(:int)
+          @msgs_left = FFI::MemoryPointer.new(:int)
         end
       end
 
@@ -44,14 +118,28 @@ module Ethon
       #   multi.perform
       def perform
         ensure_execution_mode(:perform)
+        ensure_not_cleaned_up!
+        ensure_not_executing!
 
         Ethon.logger.debug(STARTED_MULTI)
-        while ongoing?
-          run
-          timeout = get_timeout
-          next if timeout == 0
-          reset_fds
-          set_fds(timeout)
+        @executing = true
+        begin
+          while ongoing?
+            run
+            timeout = get_timeout
+            next if timeout == 0
+            case @wait_strategy
+            when :poll
+              poll(timeout)
+            when :wait
+              wait(timeout)
+            else
+              reset_fds
+              set_fds(timeout)
+            end
+          end
+        ensure
+          @executing = false
         end
         Ethon.logger.debug(PERFORMED_MULTI)
         nil
@@ -67,12 +155,63 @@ module Ethon
       # @deprecated It is no longer necessary to call prepare.
       def prepare
         Ethon.logger.warn(
-          "ETHON: It is no longer necessay to call "+
-          "Multi#prepare. Its going to be removed "+
+          "ETHON: It is no longer necessary to call "+
+          "Multi#prepare. It's going to be removed "+
           "in future versions."
         )
       end
 
+      # Wait for activity on the multi's sockets.
+      #
+      # @example Wait up to one second.
+      #   multi.wait(1000)
+      #
+      # @param [ Integer ] timeout_ms Maximum time to wait in milliseconds.
+      #
+      # @return [ Integer ] Number of file descriptors with activity.
+      def wait(timeout_ms = 1000)
+        unless Curl.respond_to?(:multi_wait)
+          raise NotImplementedError, "curl_multi_wait is not supported by this libcurl"
+        end
+
+        numfds = (@numfds ||= ::FFI::MemoryPointer.new(:int))
+        code = Curl.multi_wait(handle, ::FFI::Pointer::NULL, 0, timeout_ms, numfds)
+        raise Errors::MultiWait.new(code) unless code == :ok
+
+        numfds.read_int
+      end
+
+      # Wait for activity using curl_multi_poll.
+      #
+      # @param [ Integer ] timeout_ms Maximum time to wait in milliseconds.
+      #
+      # @return [ Integer ] Number of file descriptors with activity.
+      def poll(timeout_ms = 1000)
+        unless Curl.respond_to?(:multi_poll)
+          raise NotImplementedError, "curl_multi_poll is not supported by this libcurl"
+        end
+
+        numfds = (@numfds ||= ::FFI::MemoryPointer.new(:int))
+        code = Curl.multi_poll(handle, ::FFI::Pointer::NULL, 0, timeout_ms, numfds)
+        raise Errors::MultiPoll.new(code) unless code == :ok
+
+        numfds.read_int
+      end
+
+      # Wake up a multi handle blocked in curl_multi_poll.
+      #
+      # @return [ Symbol ] The Curl multi return code.
+      def wakeup
+        unless Curl.respond_to?(:multi_wakeup)
+          raise NotImplementedError, "curl_multi_wakeup is not supported by this libcurl"
+        end
+
+        code = Curl.multi_wakeup(handle)
+        raise Errors::MultiWakeup.new(code) unless code == :ok
+
+        code
+      end
+
       # Continue execution with an external IO loop.
       #
       # @example When no sockets are ready yet, or to begin.
@@ -84,9 +223,16 @@ module Ethon
       # @example When a socket is readable and writable
       #   multi.socket_action(io_object, [:in, :out])
       #
+      # @param [ IO, Integer, nil ] io The socket to act on, or nil for the
+      #   timeout case.
+      # @param [ Integer, Symbol, Array ] readiness CURL_CSELECT_* readiness
+      #   bits, either as a number or as symbols such as :in and :out.
+      #
       # @return [ Symbol ] The Curl.multi_socket_action return code.
       def socket_action(io = nil, readiness = 0)
         ensure_execution_mode(:socket_action)
+        ensure_not_cleaned_up!
+        ensure_not_executing!
 
         fd = if io.nil?
           ::Ethon::Curl::SOCKET_TIMEOUT
@@ -96,12 +242,17 @@ module Ethon
           io.fileno
         end
 
-        code = Curl.multi_socket_action(handle, fd, readiness, @running_count_pointer)
-        @running_count = @running_count_pointer.read_int
+        @executing = true
+        begin
+          code = Curl.multi_socket_action(handle, fd, readiness, @running_count_pointer)
+          @running_count = @running_count_pointer.read_int
 
-        check
+          check
 
-        code
+          code
+        ensure
+          @executing = false
+        end
       end
 
       # Return whether the multi still contains requests or not.
@@ -111,11 +262,86 @@ module Ethon
       #
       # @return [ Boolean ] True if ongoing, else false.
       def ongoing?
-        easy_handles.size > 0 || (!defined?(@running_count) || running_count > 0)
+        stack.size > 0 || (running_count || 0) > 0
+      end
+
+      # Associate an arbitrary pointer with a socket in the multi handle.
+      #
+      # @param [ Integer, IO ] io The socket descriptor.
+      # @param [ FFI::Pointer, nil ] pointer The pointer to associate, or nil
+      #   to clear it.
+      #
+      # @return [ Symbol ] The Curl multi return code.
+      def assign(io, pointer)
+        unless pointer.nil? || pointer.is_a?(::FFI::Pointer)
+          raise Errors::InvalidValue.new(:assign, pointer)
+        end
+        # A freed wrapper still looks like a pointer, but libcurl would keep
+        # the dangling address.
+        if pointer.is_a?(Ethon::Curl::OwnedPointer)
+          pointer.check_pid!
+        end
+
+        socket = io.respond_to?(:fileno) ? io.fileno : io
+        # FFI would reject a non-integer descriptor with a TypeError instead
+        # of the error Ethon uses for invalid option values.
+        unless socket.is_a?(Integer)
+          raise Errors::InvalidValue.new(:assign, io)
+        end
+        code = Curl.multi_assign(handle, socket, pointer)
+        raise Errors::MultiAssign.new(code) unless code == :ok
+
+        # libcurl keeps the pointer until the socket is removed or assigned
+        # again, so keep it alive from Ruby as well.
+        if pointer.nil? || pointer.null?
+          (@assigned_sockets ||= {}).delete(socket)
+        else
+          (@assigned_sockets ||= {})[socket] = pointer
+        end
+
+        code
+      end
+
+      # Return the easy handles currently attached natively.
+      #
+      # @return [ Array<FFI::Pointer> ] Native easy handles.
+      def native_handles
+        unless Curl.respond_to?(:multi_get_handles)
+          raise NotImplementedError, "curl_multi_get_handles is not supported by this libcurl"
+        end
+
+        pointer = Curl.multi_get_handles(handle)
+        raise NoMemoryError, "curl_multi_get_handles failed" if pointer.null?
+
+        begin
+          handles = []
+          offset = 0
+          loop do
+            easy = pointer.get_pointer(offset)
+            break if easy.null?
+
+            handles << easy
+            offset += ::FFI.type_size(:pointer)
+          end
+          handles
+        ensure
+          Curl.free(pointer)
+        end
       end
 
       private
 
+      def ensure_not_cleaned_up!
+        return unless @cleaned_up
+        raise Errors::EthonError, "Multi has been cleaned up"
+      end
+
+      def ensure_not_executing!
+        return unless @executing
+
+        raise Errors::EthonError, "Multi is currently executing; wait for it to finish"
+      end
+
       # Get timeout.
       #
       # @example Get timeout.
@@ -158,7 +384,8 @@ module Ethon
         raise Errors::MultiFdset.new(code) unless code == :ok
         max_fd = @max_fd.read_int
         if max_fd == -1
-          sleep(0.001)
+          # No sockets are ready; wait for the timer instead of spinning.
+          sleep(timeout / 1000.0)
         else
           @timeval[:sec] = timeout / 1000
           @timeval[:usec] = (timeout * 1000) % 1000000
@@ -177,16 +404,31 @@ module Ethon
       #
       # @return [ void ]
       def check
-        msgs_left = ::FFI::MemoryPointer.new(:int)
+        msgs_left = (@msgs_left ||= ::FFI::MemoryPointer.new(:int))
         while true
           msg = Curl.multi_info_read(handle, msgs_left)
           break if msg.null?
           next if msg[:code] != :done
-          easy = easy_handles.find{ |e| e.handle == msg[:easy_handle] }
+          easy = stack.find{ |e| e.handle == msg[:easy_handle] }
+          # The handle may have been removed between the transfer finishing and
+          # the completion message being read.
+          next unless easy
           easy.return_code = msg[:data][:code]
           Ethon.logger.debug { "ETHON:         performed #{easy.log_inspect}" }
-          delete(easy)
+          # delete finishes the transfer's response stream. A failed close or
+          # flush must not keep on_complete from running; raise it afterwards.
+          teardown_error = nil
+          begin
+            delete(easy)
+          rescue Exception => exception
+            # A handle that libcurl still owns must stay tracked for cleanup.
+            raise if stack.include?(easy)
+
+            teardown_error = exception
+          end
           easy.complete
+          easy.raise_callback_error
+          raise teardown_error if teardown_error
         end
       end
 
@@ -197,8 +439,10 @@ module Ethon
       #
       # @return [ void ]
       def run
-        running_count_pointer = FFI::MemoryPointer.new(:int)
+        running_count_pointer = (@running_count_pointer ||= ::FFI::MemoryPointer.new(:int))
         begin code = trigger(running_count_pointer) end while code == :call_multi_perform
+        raise Errors::MultiPerform.new(code) unless code == :ok
+
         check
       end
 
@@ -221,7 +465,7 @@ module Ethon
       #
       # @return [ Integer ] Number running requests.
       def running_count
-        @running_count ||= nil
+        @running_count
       end
     end
   end