curb 1.3.5 → 1.3.7

data/ext/curb_multi.h

@@ -28,6 +28,7 @@ extern const rb_data_type_t ruby_curl_multi_data_type;
 
 void init_curb_multi();
 void rb_curl_multi_forget_easy(ruby_curl_multi *rbcm, void *rbce_ptr);
+CURLMcode rb_curl_multi_detach_easy(ruby_curl_multi *rbcm, void *rbce_ptr);
 
 
 #endif

data/ext/extconf.rb

@@ -298,6 +298,8 @@ have_constant "curlopt_sockoptfunction"
 have_constant "curlopt_sockoptdata"
 have_constant "curlopt_opensocketfunction"
 have_constant "curlopt_opensocketdata"
+have_constant "curlopt_prereqfunction"
+have_constant "curlopt_prereqdata"
 
 # Deprecated constants (still check for them for backward compat)
 have_constant "curlopt_ioctlfunction"
@@ -391,6 +393,7 @@ have_constant "curlopt_socks5_gssapi_nec"
 have_constant "curlopt_interface"
 have_constant "curlopt_localport"
 have_constant "curlopt_dns_cache_timeout"
+have_constant "curlopt_dns_servers"
 have_constant "curlopt_dns_use_global_cache"
 have_constant "curlopt_buffersize"
 have_constant "curlopt_port"
@@ -459,6 +462,9 @@ have_constant "curlopt_ftp_use_eprt"
 have_constant "curlopt_ftp_use_epsv"
 have_constant "curlopt_ftp_use_pret"
 have_constant "curlopt_ftp_create_missing_dirs"
+have_constant "curlftp_create_dir_none"
+have_constant "curlftp_create_dir"
+have_constant "curlftp_create_dir_retry"
 have_constant "curlopt_ftp_response_timeout"
 have_constant "curlopt_ftp_alternative_to_user"
 have_constant "curlopt_ftp_skip_pasv_ip"
@@ -531,6 +537,7 @@ have_constant "curlusessl_none"
 have_constant "curlusessl_try"
 have_constant "curlusessl_control"
 have_constant "curlusessl_all"
+have_constant "curlopt_connect_to"
 have_constant "curlopt_resolve"
 have_constant "curlopt_request_target"
 have_constant "curlopt_sslcert"
@@ -558,6 +565,10 @@ have_constant :CURL_SSLVERSION_TLSv1_2
 have_constant :CURL_SSLVERSION_TLSv1_3
 
 have_constant "curlopt_ssl_verifypeer"
+have_constant "curlopt_doh_url"
+have_constant "curlopt_doh_ssl_verifypeer"
+have_constant "curlopt_doh_ssl_verifyhost"
+have_constant "curlopt_doh_ssl_verifystatus"
 have_constant "curlopt_cainfo"
 have_constant "curlopt_issuercert"
 have_constant "curlopt_capath"
@@ -692,6 +703,7 @@ have_header('ruby/fiber/scheduler.h')
 have_func('rb_fiber_scheduler_current', 'ruby/fiber/scheduler.h')
 have_func('rb_fiber_scheduler_io_wait', 'ruby/fiber/scheduler.h')
 have_func('rb_fiber_scheduler_io_select', 'ruby/fiber/scheduler.h')
+have_func('rb_fiber_scheduler_kernel_sleep', 'ruby/fiber/scheduler.h')
 have_func('rb_io_stdio_file')
 have_func('curl_multi_wait')
 have_func('curl_multi_socket_action')

data/lib/curl/download.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'tempfile'
+
+module Curl
+  class DownloadTargetExistsError < Errno::EEXIST; end
+  DOWNLOAD_OPTION_KEYS = [:download_dir, :overwrite].freeze
+
+  class << self
+    def download_options_hash?(value)
+      value.is_a?(Hash) && value.keys.all? { |key| DOWNLOAD_OPTION_KEYS.include?(key) }
+    end
+
+    def normalize_download_arguments(filename, options)
+      if download_options_hash?(filename) && options.empty?
+        options = filename
+        filename = nil
+      end
+
+      [filename, options || {}]
+    end
+
+    def parse_download_options(options)
+      raise ArgumentError, "download options must be a Hash" unless options.is_a?(Hash)
+
+      options = options.dup
+      download_dir = options.delete(:download_dir)
+      overwrite = options.key?(:overwrite) ? !!options.delete(:overwrite) : false
+      raise ArgumentError, "unsupported download option(s): #{options.keys.join(', ')}" unless options.empty?
+
+      [download_dir, overwrite]
+    end
+
+    def resolve_download_output(url, filename = nil, options = {})
+      filename, options = normalize_download_arguments(filename, options)
+      download_dir, overwrite = parse_download_options(options)
+
+      if filename.is_a?(IO)
+        filename.binmode if filename.respond_to?(:binmode)
+        return [nil, filename, false, overwrite]
+      end
+
+      path = if download_dir
+        safe_download_path(url, download_dir, filename: filename)
+      elsif filename.nil?
+        safe_download_path(url)
+      else
+        filename.to_s
+      end
+
+      [path, nil, true, overwrite]
+    end
+
+    def prepare_download_output(url, filename = nil, options = {})
+      path, io, safe_output, overwrite = resolve_download_output(url, filename, options)
+      return [path, io, safe_output] unless safe_output
+
+      [path, open_safe_download_output(path, overwrite: overwrite), true]
+    end
+
+    def download_filename_from_url(url)
+      path = begin
+        URI.parse(url.to_s).path
+      rescue URI::InvalidURIError
+        url.to_s.split(/\?/, 2).first
+      end
+
+      basename = File.basename(path.to_s)
+      validate_download_filename!(basename)
+    end
+
+    def safe_download_path(url, destination_dir = Dir.pwd, filename: nil)
+      dir = File.expand_path(destination_dir.to_s)
+      raise ArgumentError, "download destination directory does not exist: #{destination_dir}" unless File.directory?(dir)
+
+      File.join(dir, filename.nil? ? download_filename_from_url(url) : validate_download_filename!(filename))
+    end
+
+    def validate_download_filename!(filename)
+      filename = filename.to_s
+      invalid = filename.empty? ||
+                filename == '.' ||
+                filename == '..' ||
+                filename == File::SEPARATOR ||
+                filename.start_with?('.') ||
+                filename.include?("\0") ||
+                filename.include?(File::SEPARATOR) ||
+                filename.include?('\\') ||
+                filename.match?(/\A[A-Za-z]:/) ||
+                (File::ALT_SEPARATOR && filename.include?(File::ALT_SEPARATOR))
+
+      raise ArgumentError, "unsafe download filename derived from URL: #{filename.inspect}" if invalid
+
+      filename
+    end
+
+    def open_safe_download_output(path, overwrite: false)
+      SafeDownloadOutput.new(path, overwrite: overwrite)
+    end
+  end
+
+  class SafeDownloadOutput
+    def initialize(path, overwrite: false)
+      @path = File.expand_path(path.to_s)
+      @overwrite = overwrite
+      raise DownloadTargetExistsError, @path if !@overwrite && File.exist?(@path)
+
+      @tmp = Tempfile.new(['.curb-download-', '.tmp'], File.dirname(@path))
+      @tmp.binmode
+      @closed = false
+    end
+
+    attr_reader :path
+
+    def write(data)
+      @tmp.write(data)
+    end
+
+    def <<(data)
+      write(data)
+    end
+
+    def close(success = false)
+      return if @closed
+
+      @tmp.flush unless @tmp.closed?
+      @tmp.close unless @tmp.closed?
+      install_tmp_file if success
+    ensure
+      @tmp.close! if @tmp
+      @closed = true
+    end
+
+    private
+
+    def install_tmp_file
+      if @overwrite
+        File.rename(@tmp.path, @path)
+      else
+        File.link(@tmp.path, @path)
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    rescue NotImplementedError
+      install_tmp_file_by_copy
+    end
+
+    def install_tmp_file_by_copy
+      flags = File::WRONLY | File::CREAT | File::EXCL
+      flags |= File::BINARY if File.const_defined?(:BINARY)
+
+      File.open(@path, flags) do |dest|
+        File.open(@tmp.path, 'rb') { |src| IO.copy_stream(src, dest) }
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    end
+  end
+end

data/lib/curl/easy.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'curl/download'
 module Curl
   class Easy
     class << self
@@ -83,10 +84,12 @@ module Curl
     
     alias_method :_curb_native_close, :close
     alias_method :_curb_native_multi_set, :multi=
+    alias_method :_curb_native_reset, :reset
     
     def close
       previous_multi = self.multi
       result = _curb_native_close
+      __curb_clear_safety_override!
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       result
     end
@@ -94,12 +97,22 @@ module Curl
     def multi=(multi)
       previous_multi = self.multi
       return multi if previous_multi.equal?(multi)
-    
+
+      if previous_multi && previous_multi.requests[self.object_id]
+        previous_multi.remove(self)
+      end
+
       result = _curb_native_multi_set(multi)
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       multi.__send__(:__register_idle_easy_reference, self) if multi
       result
     end
+
+    def reset
+      result = _curb_native_reset
+      __curb_clear_safety_override!
+      result
+    end
     
     alias post http_post
     alias put http_put
@@ -154,6 +167,84 @@ module Curl
       Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
     end
 
+    def allowed_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_PROTOCOLS_STR', 'CURLOPT_PROTOCOLS', protocols)
+    end
+
+    def allowed_redirect_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_REDIR_PROTOCOLS_STR', 'CURLOPT_REDIR_PROTOCOLS', protocols)
+    end
+
+    def safe_http!
+      __curb_set_safety_override!(
+        :protocols => [:http, :https],
+        :redirect_protocols => [:http, :https]
+      )
+    end
+
+    private
+
+    def __curb_set_safety_override!(options)
+      @__curb_safety_override = (defined?(@__curb_safety_override) && @__curb_safety_override) ? @__curb_safety_override.dup : {}
+      @__curb_safety_override[:protocols] = Array(options[:protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:protocols)
+      @__curb_safety_override[:redirect_protocols] = Array(options[:redirect_protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:redirect_protocols)
+      @__curb_safety_override[:max_body_bytes] = options[:max_body_bytes] if options.key?(:max_body_bytes) && options[:max_body_bytes]
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
+      self
+    end
+
+    def __curb_clear_safety_override!
+      had_override = defined?(@__curb_safety_override) && @__curb_safety_override
+      return unless had_override
+      return if frozen?
+
+      @__curb_safety_override = nil
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+    end
+
+    def __curb_safety_override
+      defined?(@__curb_safety_override) ? @__curb_safety_override : nil
+    end
+
+    def __curb_safety_override_generation
+      defined?(@__curb_safety_override_generation) ? @__curb_safety_override_generation : 0
+    end
+
+    def set_protocol_allowlist(string_option, bitmask_option, protocols)
+      protocol_names = Array(protocols).map { |protocol| protocol.to_s.downcase }
+      raise ArgumentError, "at least one protocol is required" if protocol_names.empty?
+
+      valid_names = %w[
+        dict file ftp ftps gopher gophers http https imap imaps ldap ldaps
+        mqtt pop3 pop3s rtmp rtmpe rtmps rtmpt rtmpte rtmpts rtsp scp sftp
+        smb smbs smtp smtps telnet tftp ws wss
+      ]
+
+      if Curl.const_defined?(string_option)
+        protocol_names.each do |name|
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless valid_names.include?(name)
+        end
+
+        setopt(Curl.const_get(string_option), protocol_names.join(','))
+      elsif Curl.const_defined?(bitmask_option)
+        protocol_pairs = protocol_names.map do |name|
+          const_name = "CURLPROTO_#{name.upcase}"
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless Curl.const_defined?(const_name)
+
+          [name, Curl.const_get(const_name)]
+        end
+
+        setopt(Curl.const_get(bitmask_option), protocol_pairs.inject(0) { |mask, pair| mask | pair.last })
+      else
+        raise NotImplementedError, "protocol allowlists are not supported by this libcurl"
+      end
+
+      protocols
+    end
+
+    public
+
     #
     # call-seq:
     #   easy.perform                                     => true
@@ -163,6 +254,7 @@ module Curl
     # the configured HTTP Verb.
     #
     def perform
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
       self.class.flush_deferred_multi_closes
 
       if Curl.scheduler_active? && self.multi.nil?
@@ -210,6 +302,10 @@ module Curl
         raise callback_error
       end
 
+      if respond_to?(:unsafe_destination_error) && (unsafe_destination_error = self.unsafe_destination_error)
+        raise Curl::Err::UnsafeDestinationError, unsafe_destination_error
+      end
+
       if self.last_result != 0 && self.on_failure.nil?
         err_class, err_summary = Curl::Easy.error(self.last_result)
         err_detail = self.last_error
@@ -641,11 +737,15 @@ module Curl
       end
 
       # call-seq:
-      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   Curl::Easy.download(url, filename = nil, options = {}) { |curl| ... }
       #
       # Stream the specified url (via perform) and save the data directly to the
-      # supplied filename (defaults to the last component of the URL path, which will
-      # usually be the filename most simple urls).
+      # supplied filename. The destination is written through a temporary file and
+      # existing files are not overwritten unless <tt>:overwrite => true</tt> is
+      # passed. When filename is omitted, the destination is safely derived from the
+      # last URL path component in the current directory. Pass
+      # <tt>:download_dir</tt> to treat the filename as a basename inside a trusted
+      # directory and reject absolute, parent-directory, dotfile, and nested names.
       #
       # If a block is supplied, it will be passed the curl instance prior to the
       # perform call.
@@ -658,16 +758,11 @@ module Curl
       # returns a size that differs from the data chunk size - in this case, the
       # offending chunk will *not* be written to the file, the file will be closed,
       # and a Curl::Err::AbortedByCallbackError will be raised.
-      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+      def download(url, filename = nil, download_options = {}, &blk)
         curl = Curl::Easy.new(url, &blk)
+        _download_path, output, safe_output = Curl.prepare_download_output(url, filename, download_options)
 
-        output = if filename.is_a? IO
-          filename.binmode if filename.respond_to?(:binmode)
-          filename
-        else
-          File.open(filename, 'wb')
-        end
-
+        performed = false
         begin
           old_on_body = curl.on_body do |data|
             result = old_on_body ?  old_on_body.call(data) : data.length
@@ -675,8 +770,13 @@ module Curl
             result
           end
           curl.perform
+          performed = true
         ensure
-          output.close rescue IOError
+          if safe_output
+            output.close(performed)
+          else
+            output.close rescue IOError
+          end
         end
 
         return curl