ethon 0.17.0 → 0.19.0

data/lib/ethon/easy/callbacks.rb

@@ -20,13 +20,20 @@ module Ethon
       # @example Set callbacks.
       #   easy.set_callbacks
       def set_callbacks
-        Curl.set_option(:writefunction, body_write_callback, handle)
-        Curl.set_option(:headerfunction, header_write_callback, handle)
-        Curl.set_option(:debugfunction, debug_callback, handle)
-        @response_body = String.new
+        install_callbacks(handle)
+        reset_transfer_state
+      end
+
+      # Reset the state that belongs to a single transfer. Reusing an Easy must
+      # replace the previous response instead of appending to it.
+      def reset_transfer_state
+        rewind_request_body
+        @response_body = @response_body_io ? nil : String.new
         @response_headers = String.new
         @headers_called = false
         @debug_info = Ethon::Easy::DebugInfo.new
+        @request_body_read = 0
+        @return_code = nil
       end
 
       # Returns the body write callback.
@@ -37,10 +44,19 @@ module Ethon
       # @return [ Proc ] The callback.
       def body_write_callback
         @body_write_callback ||= proc do |stream, size, num, object|
-          headers
-          result = body(chunk = stream.read_string(size * num))
-          @response_body << chunk if result == :unyielded
-          result != :abort ? size * num : -1
+          begin
+            headers_user_callback_result = headers
+            result = body(chunk = stream.read_string(size * num))
+            if @response_body_io && result != :abort && headers_user_callback_result != :abort
+              write_response_body(@response_body_io, chunk)
+            elsif result == :unyielded && !@response_body_io && headers_user_callback_result != :abort
+              @response_body << chunk
+            end
+            (result != :abort && headers_user_callback_result != :abort) ? size * num : -1
+          rescue Exception => e
+            @callback_error ||= e
+            0
+          end
         end
       end
 
@@ -52,14 +68,21 @@ module Ethon
       # @return [ Proc ] The callback.
       def header_write_callback
         @header_write_callback ||= proc {|stream, size, num, object|
-          result = headers
-          @response_headers << stream.read_string(size * num)
-          result != :abort ? size * num : -1
+          begin
+            @response_headers << stream.read_string(size * num)
+            size * num
+          rescue Exception => e
+            @callback_error ||= e
+            0
+          end
         }
       end
 
-      # Returns the debug callback. This callback is currently used
-      # write the raw http request headers.
+      # This callback is called for every debug message libcurl emits while
+      # verbose mode is enabled.
+      #
+      # Messages are written through +Ethon.logger+ at the debug level. The
+      # user observer registered with #debugfunction is called as well.
       #
       # @example Return the callback.
       #   easy.debug_callback
@@ -67,19 +90,37 @@ module Ethon
       # @return [ Proc ] The callback.
       def debug_callback
         @debug_callback ||= proc {|handle, type, data, size, udata|
-          message = data.read_string(size)
-          @debug_info.add type, message
-          print message unless [:data_in, :data_out].include?(type)
+          begin
+            message = data.read_string(size)
+            @debug_info.add type, message
+
+            if [:ssl_data_in, :ssl_data_out].include?(type)
+              log_debug("[#{size} bytes data]")
+            elsif [:data_in, :data_out].include?(type)
+              # Body data stays in #debug_info; logging it would duplicate the
+              # response body in the application log.
+            else
+              log_debug(message)
+            end
+            @user_debug_callback.call(handle, type, data, size, udata) if @user_debug_callback
+          rescue Exception => e
+            @callback_error ||= e
+          end
           0
         }
       end
 
       def set_progress_callback
-        if Curl.version_info[:version] >= "7.32.0"
-          Curl.set_option(:xferinfofunction, progress_callback, handle)
+        # CURLOPT_XFERINFOFUNCTION was added in libcurl 7.32.0.
+        option = if Curl.version_at_least?("7.32.0")
+          :xferinfofunction
         else
-          Curl.set_option(:progressfunction, progress_callback, handle)
+          :progressfunction
         end
+        Curl.set_option(option, progress_callback, handle)
+        # Remember which callback is effective so a duplicate can restore it.
+        @progress_source = :internal
+        @internal_progress_option = option
       end
 
       # Returns the progress callback.
@@ -89,10 +130,34 @@ module Ethon
       #
       # @return [ Proc ] The callback.
       def progress_callback
-        @progress_callback ||= proc { |_, dltotal, dlnow, ultotal, ulnow|
-          progress(dltotal, dlnow, ultotal, ulnow)
-          0
-        }
+        @progress_callback ||= proc do |_, dltotal, dlnow, ultotal, ulnow|
+          begin
+            progress(dltotal, dlnow, ultotal, ulnow)
+            0
+          rescue Exception => e
+            @callback_error ||= e
+            0
+          end
+        end
+      end
+
+      # A completion callback may start the next transfer on this Easy. Keep
+      # the finished transfer's callback error so the caller still sees it;
+      # the next transfer starts without it once the error is raised.
+      def clear_callback_error
+        @callback_error = nil unless @completing
+      end
+
+      # The transfer teardown in #perform, #complete, and Multi#check already
+      # finishes the response stream. Finishing it here again could close a
+      # stream a completion callback installed for the next transfer, or
+      # replace the callback error with a teardown error.
+      def raise_callback_error
+        return unless @callback_error
+
+        error = @callback_error
+        @callback_error = nil
+        raise error
       end
 
       # Set the read callback. This callback is used by libcurl to
@@ -104,35 +169,52 @@ module Ethon
       # @param [ String ] body The body.
       def set_read_callback(body)
         @request_body_read = 0
-        readfunction do |stream, size, num, object|
-          size = size * num
-          body_size = if body.respond_to?(:bytesize)
-            body.bytesize
-          elsif body.respond_to?(:size)
-            body.size
-          elsif body.is_a?(File)
-            File.size(body.path)
-          end
+        @request_body = body
+        # Remember where a seekable body starts so reusing the Easy replays the
+        # same range instead of finding the IO at EOF.
+        @request_body_start = begin
+          body.pos if body.respond_to?(:seek) && body.respond_to?(:pos)
+        rescue IOError, SystemCallError
+          nil
+        end
+        @read_callback = proc do |stream, size, num, object|
+          begin
+            size = size * num
+            remaining = if body.respond_to?(:byteslice)
+              body.bytesize - @request_body_read
+            elsif body.respond_to?(:read)
+              # IO-like bodies advance their own position.
+              body.size - body.pos if body.respond_to?(:size) && body.respond_to?(:pos)
+            elsif body.respond_to?(:size)
+              body.size - @request_body_read
+            end
 
-          left = body_size - @request_body_read
-          size = left if size > left
+            size = remaining if remaining && size > remaining
 
-          if size > 0
-            chunk = if body.respond_to?(:byteslice)
-              body.byteslice(@request_body_read, size)
-            elsif body.respond_to?(:read)
-              body.read(size)
-            else
-              body[@request_body_read, size]
+            chunk = if size > 0
+              if body.respond_to?(:byteslice)
+                body.byteslice(@request_body_read, size)
+              elsif body.respond_to?(:read)
+                body.read(size)
+              else
+                body[@request_body_read, size]
+              end
             end
 
-            stream.write_string(
-              chunk, size
-            )
-            @request_body_read += size
+            # libcurl expects the number of bytes actually copied into the
+            # buffer. IO-like bodies may return fewer bytes than requested.
+            chunk_size = chunk ? chunk.bytesize : 0
+            if chunk_size > 0
+              stream.write_string(chunk, chunk_size)
+              @request_body_read += chunk_size
+            end
+            chunk_size
+          rescue Exception => e
+            @callback_error ||= e
+            0
           end
-          size
         end
+        readfunction(&@read_callback)
       end
 
       # Returns the body read callback.
@@ -144,6 +226,56 @@ module Ethon
       def read_callback
         @read_callback
       end
+
+      private
+
+      # IO#write may accept fewer bytes than it was given, especially for
+      # nonblocking or wrapper IOs. libcurl must only see the chunk as handled
+      # once every byte reached the destination. IOs that return nil (or a
+      # non-integer) keep the historical "wrote everything" behavior.
+      def write_response_body(io, chunk)
+        offset = 0
+        total = chunk.bytesize
+        while offset < total
+          pending = offset.zero? ? chunk : chunk.byteslice(offset, total - offset)
+          written = io.write(pending)
+          return unless written.is_a?(Integer)
+
+          raise IOError, "response body write made no progress" if written <= 0
+          offset += written
+        end
+      end
+
+      # Install the callbacks that fill the Ruby-side response state. The
+      # handle is passed explicitly so callers do not have to go through
+      # #handle, which would create a handle as a side effect.
+      def install_callbacks(target)
+        Curl.set_option(:writefunction, body_write_callback, target)
+        Curl.set_option(:headerfunction, header_write_callback, target)
+        Curl.set_option(:debugfunction, debug_callback, target)
+      end
+
+      # Log a libcurl debug message through Ethon.logger. The default level is
+      # INFO, so applications opt in by lowering the logger level.
+      def log_debug(message)
+        return if message.empty?
+
+        logger = Ethon.logger
+        return unless logger.debug?
+
+        logger.debug { message.sub(/\n+\z/, "") }
+      end
+
+      # Seek a request body back to where it was when the callback was
+      # installed. Pipes and other non-seekable bodies are left alone.
+      def rewind_request_body
+        return unless @request_body_start
+        return unless @request_body.respond_to?(:seek)
+
+        @request_body.seek(@request_body_start)
+      rescue IOError, SystemCallError
+        nil
+      end
     end
   end
 end

data/lib/ethon/easy/features.rb

@@ -24,7 +24,10 @@ module Ethon
         !!(Curl.version_info[:features] & Curl::VERSION_ASYNCHDNS)
       end
 
-      alias :supports_timeout_ms? :supports_asynch_dns?
+      # CURLOPT_TIMEOUT_MS was added in libcurl 7.16.2.
+      def supports_timeout_ms?
+        Curl.version_at_least?("7.16.2")
+      end
 
     end
   end

data/lib/ethon/easy/files.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+module Ethon
+  class Easy
+    # This module contains logic for streaming response bodies to Ruby IOs.
+    module Files
+      # Stream the response body to a path or writable Ruby object. Ethon closes
+      # files it opens from paths; caller-provided objects remain caller-owned.
+      # Pass nil to restore in-memory response buffering.
+      def file=(destination)
+        # Resolve the handle first. Opening a path truncates it, so a cleaned
+        # up or fork-inherited handle must not touch the destination before the
+        # error is raised.
+        handle
+
+        # Install the new destination before releasing the old one. A failed
+        # open must leave the previous destination and its buffered response
+        # usable instead of leaving the Easy pointed at a path that cannot be
+        # opened.
+        # Pathname-like objects may also implement #write. Prefer the path when
+        # the object is not an IO (File and Tempfile expose #fileno).
+        if destination.respond_to?(:to_path) && !destination.respond_to?(:fileno)
+          replace_response_body_path(destination.to_path)
+        elsif destination.respond_to?(:write)
+          replace_response_body_io(destination)
+        elsif destination
+          replace_response_body_path(destination.to_s)
+        else
+          clear_response_body_io
+        end
+
+        @response_body = @response_body_io ? nil : String.new
+        destination
+      end
+
+      alias writedata= file=
+
+      def prepare_response_body_io
+        return unless @response_body_path && !@response_body_io
+
+        @response_body_io = File.open(@response_body_path, "wb")
+        @owns_response_body_io = true
+      end
+
+      def close_response_body_io
+        @response_body_io.close if @owns_response_body_io && @response_body_io && !@response_body_io.closed?
+        @response_body_io = nil
+        @owns_response_body_io = false
+      end
+
+      def clear_response_body_io
+        close_response_body_io
+        @response_body_path = nil
+      end
+
+      # Flush buffered writes without closing a caller-owned IO. Response
+      # callbacks and readers should see the complete body before the transfer
+      # is treated as finished.
+      def flush_response_body_io
+        io = @response_body_io
+        return unless io
+        return if io.respond_to?(:closed?) && io.closed?
+
+        io.flush if io.respond_to?(:flush)
+      end
+
+      # Finish the destination used by one transfer. A completion callback may
+      # already have installed the destination for the next transfer; only the
+      # stream this transfer wrote to belongs to it.
+      #
+      # @param [ IO, nil ] io The destination that transfer used.
+      def finish_response_body_io(io = @response_body_io)
+        return unless @response_body_io && @response_body_io.equal?(io)
+
+        if @owns_response_body_io
+          close_response_body_io
+        else
+          flush_response_body_io
+        end
+      end
+
+      private
+
+      # Open the new path first. If that fails, or if the old destination
+      # cannot be released, the previous state stays in place.
+      def replace_response_body_path(path)
+        io = File.open(path, "wb")
+        begin
+          clear_response_body_io
+        rescue Exception
+          close_quietly(io)
+          raise
+        end
+
+        @response_body_path = path
+        @response_body_io = io
+        @owns_response_body_io = true
+      end
+
+      def replace_response_body_io(io)
+        clear_response_body_io
+        @response_body_io = io
+        @owns_response_body_io = false
+      end
+
+      def close_quietly(io)
+        io.close unless io.closed?
+      rescue StandardError
+        # The caller is already handling the failure that kept us from
+        # installing the new destination.
+      end
+    end
+  end
+end

data/lib/ethon/easy/form.rb

@@ -57,7 +57,7 @@ module Ethon
       # @return [ Boolean ] True if form is multipart, else false.
       def multipart?
         return true if @multipart
-        query_pairs.any?{|pair| pair.respond_to?(:last) && pair.last.is_a?(Array)}
+        contains_file?(@params)
       end
 
       # Add form elements to libcurl.
@@ -65,20 +65,77 @@ module Ethon
       # @example Add form to libcurl.
       #   form.materialize
       def materialize
+        unless Curl.respond_to?(:formadd)
+          raise NotImplementedError,
+            "curl_formadd is not supported by this libcurl; use the MIME API instead"
+        end
+
+        ensure_named_fields!
         query_pairs.each { |pair| form_add(pair.first.to_s, pair.last) }
       end
 
+      # Build a MIME structure for this form.
+      #
+      # @example Build a MIME structure.
+      #   form.mime
+      #
+      # @return [ Ethon::Curl::Mime ] The MIME structure.
+      def mime
+        @mime ||= begin
+          ensure_named_fields!
+          structure = Ethon::Curl::Mime.new(@easy)
+          query_pairs.each do |pair|
+            part = structure.addpart
+            name, content = pair
+            part.name = name.to_s
+            if content.is_a?(Queryable::FileInfo)
+              part.file = content[2]
+              part.filename = content[0]
+              part.type = content[1]
+            else
+              part.data = content.to_s
+            end
+          end
+          structure
+        end
+      end
+
       private
 
-      def form_add(name, content)
-        case content
+      # A multipart part needs a field name. A raw String body is one pair and
+      # has no name, which used to produce an unnamed part through the MIME API
+      # and a NoMethodError through the form API.
+      def ensure_named_fields!
+        return unless query_pairs.any? { |pair| pair.is_a?(String) }
+
+        raise ArgumentError, "multipart bodies require a Hash of fields"
+      end
+
+      def contains_file?(value)
+        case value
+        when Hash
+          value.values.any? { |item| contains_file?(item) }
         when Array
+          value.any? { |item| contains_file?(item) }
+        else
+          value.is_a?(File) || (defined?(Tempfile) && value.is_a?(Tempfile))
+        end
+      end
+
+      def form_add(name, content)
+        name = validate_header_line(:name, name.to_s)
+        code = case content
+        when Queryable::FileInfo
+          # The form API writes these into Content-Disposition headers. Older
+          # libcurl versions do not escape them.
+          filename = validate_header_line(:filename, content[0].to_s)
+          content_type = validate_header_line(:content_type, content[1].to_s)
           Curl.formadd(first, last,
                        :form_option, :copyname, :pointer, name,
                        :form_option, :namelength, :long, name.bytesize,
                        :form_option, :file, :string, content[2],
-                       :form_option, :filename, :string, content[0],
-                       :form_option, :contenttype, :string, content[1],
+                       :form_option, :filename, :string, filename,
+                       :form_option, :contenttype, :string, content_type,
                        :form_option, :end
                       )
         else
@@ -91,6 +148,8 @@ module Ethon
                       )
         end
 
+        raise Errors::FormAdd.new(code) unless code == 0
+
         setup_garbage_collection
       end
 
@@ -100,7 +159,7 @@ module Ethon
         # to. So this results in (pseudo-c):
         #   form_data_cleanup_handler = *first
         #   curl_form_free(form_data_cleanup_handler)
-        @form_data_cleanup_handler ||= FFI::AutoPointer.new(@first.get_pointer(0), Curl.method(:formfree))
+        @form_data_cleanup_handler ||= Curl::FormData.new(@first.get_pointer(0))
       end
     end
   end

data/lib/ethon/easy/header.rb

@@ -5,14 +5,19 @@ module Ethon
     #
     # @api private
     module Header
-      # Return headers, return empty hash if none.
+      # Return the request headers, return empty hash if none.
       #
-      # @example Return the headers.
-      #   easy.headers
+      # The returned hash is an independent snapshot of the headers that were
+      # sent to libcurl. Mutating the hash or a value passed to #headers=
+      # afterwards cannot change either the handle or this snapshot, and
+      # mutating the returned hash or its values does not change the request.
       #
-      # @return [ Hash ] The headers.
-      def headers
-        @headers ||= {}
+      # @example Return the request headers.
+      #   easy.request_headers
+      #
+      # @return [ Hash ] The request headers.
+      def request_headers
+        copy_headers(@headers || {})
       end
 
       # Set the headers.
@@ -20,17 +25,25 @@ module Ethon
       # @example Set the headers.
       #   easy.headers = {'User-Agent' => 'ethon'}
       #
-      # @param [ Hash ] headers The headers.
-      def headers=(headers)
-        headers ||= {}
-        header_list = nil
-        headers.each do |k, v|
-          header_list = Curl.slist_append(header_list, compose_header(k,v))
+      # @param [ Hash ] value The headers.
+      def headers=(value)
+        # Snapshot before building the list so the stored value is exactly the
+        # set that libcurl accepted. Copy the values too: mutating a
+        # caller-owned header value in place would otherwise change the
+        # snapshot.
+        headers = copy_headers(value || {})
+        list = build_slist(headers)
+        begin
+          Curl.set_option(:httpheader, list, handle)
+        rescue Exception
+          Curl.slist_free_all(list) if list
+          raise
         end
-        Curl.set_option(:httpheader, header_list, handle)
 
-        @header_list = header_list && FFI::AutoPointer.new(header_list, Curl.method(:slist_free_all))
+        @headers = headers.freeze
+        @header_list = list && Curl::Slist.new(list)
       end
+      alias request_headers= headers=
 
       # Return header_list.
       #
@@ -54,7 +67,36 @@ module Ethon
       #
       # @return [ String ] The composed header.
       def compose_header(key, value)
-        Util.escape_zero_byte("#{key}: #{value}")
+        header = if key.to_s.end_with?(';') && value.to_s.empty?
+          key.to_s
+        else
+          "#{key}: #{value}"
+        end
+        header = Util.escape_zero_byte(header)
+        # libcurl sends custom headers verbatim, so CR/LF would inject
+        # additional headers into the request.
+        Util.validate_header_line(:headers, header)
+      end
+
+      private
+
+      # Copy a header hash so callers and the Easy cannot change each other's
+      # header values in place. Values that cannot be copied are kept as they
+      # are; header values are normally strings.
+      def copy_headers(headers)
+        headers.each_with_object({}) do |(key, value), copy|
+          copy[key] = begin
+            value.dup
+          rescue TypeError
+            value
+          end
+        end
+      end
+
+      # Build a curl_slist for the headers. Compose first so an invalid header
+      # does not leave a partial native list behind.
+      def build_slist(headers)
+        Curl::Slist.build(headers.map { |key, value| compose_header(key, value) })
       end
     end
   end