curb 1.3.5 → 1.3.6

data/ext/curb_easy.h

@@ -11,6 +11,18 @@
 
 #include <curl/easy.h>
 
+#define CURB_NETWORK_POLICY_NONE 0
+#define CURB_NETWORK_POLICY_PUBLIC 1
+
+#define CURB_CIDR_FAMILY_IPV4 4
+#define CURB_CIDR_FAMILY_IPV6 6
+
+typedef struct {
+  unsigned char family;
+  unsigned char prefix_bits;
+  unsigned char address[16];
+} curb_cidr_rule;
+
 #ifdef CURL_VERSION_SSL
 #if LIBCURL_VERSION_NUM >= 0x070b00
 #  if LIBCURL_VERSION_NUM <= 0x071004
@@ -38,6 +50,7 @@ typedef struct {
 
   /* Buffer for error details from CURLOPT_ERRORBUFFER */
   char err_buf[CURL_ERROR_SIZE];
+  char unsafe_destination_error[CURL_ERROR_SIZE];
 
   VALUE self; /* owning Ruby object */
   VALUE opts; /* rather then allocate everything we might need to store, allocate a Hash and only store objects we actually use... */
@@ -67,6 +80,7 @@ typedef struct {
   long ftp_filemethod;
   long http_version;
   unsigned short resolve_mode;
+  unsigned short network_policy;
 
   /* bool flags */
   char proxy_tunnel;
@@ -83,13 +97,25 @@ typedef struct {
   char cookielist_engine_enabled; /* track if CURLOPT_COOKIELIST was used with a non-command to enable engine */
   char ignore_content_length;
   char callback_active;
+  char unsafe_destination_blocked;
+  char allow_proxy;
+  char allow_unix_socket;
+  char forbid_reuse_set;
+  unsigned int native_active;
+  long forbid_reuse;
 
   struct curl_slist *curl_headers;
   struct curl_slist *curl_proxy_headers;
   struct curl_slist *curl_ftp_commands;
   struct curl_slist *curl_resolve;
+  struct curl_slist *curl_connect_to;
+  curb_cidr_rule *network_allowed_cidr_rules;
+  char **network_allowed_hosts;
 
   unsigned long multi_attachment_generation;
+  curl_off_t downloaded_body_bytes;
+  size_t network_allowed_cidr_rule_count;
+  size_t network_allowed_host_count;
   int last_result; /* last result code from multi loop */
 
 } ruby_curl_easy;

data/ext/curb_errors.c

@@ -110,6 +110,7 @@ VALUE eCurlErrSSLShutdownFailed;
 VALUE eCurlErrAgain;
 VALUE eCurlErrSSLCRLBadfile;
 VALUE eCurlErrSSLIssuerError;
+VALUE eCurlErrUnsafeDestination;
 
 /* multi errors */
 VALUE mCurlErrFailedInit;
@@ -699,6 +700,7 @@ void init_curb_errors() {
   eCurlErrSSLCacertBadfile   = rb_define_class_under(mCurlErr, "SSLCaertBadFile", eCurlErrError);
   eCurlErrSSLCRLBadfile      = rb_define_class_under(mCurlErr, "SSLCRLBadfile", eCurlErrError);
   eCurlErrSSLIssuerError     = rb_define_class_under(mCurlErr, "SSLIssuerError", eCurlErrError);
+  eCurlErrUnsafeDestination  = rb_define_class_under(mCurlErr, "UnsafeDestinationError", eCurlErrError);
   eCurlErrSSLShutdownFailed  = rb_define_class_under(mCurlErr, "SSLShutdownFailed", eCurlErrError);
   eCurlErrSSH                = rb_define_class_under(mCurlErr, "SSH", eCurlErrError);
 

data/ext/curb_errors.h

@@ -106,6 +106,7 @@ extern VALUE eCurlErrSSLShutdownFailed;
 extern VALUE eCurlErrAgain;
 extern VALUE eCurlErrSSLCRLBadfile;
 extern VALUE eCurlErrSSLIssuerError;
+extern VALUE eCurlErrUnsafeDestination;
 
 /* multi errors */
 extern VALUE mCurlErrFailedInit;

data/ext/curb_multi.c

@@ -74,6 +74,7 @@ extern VALUE mCurl;
 static VALUE idCall;
 static ID id_deferred_exception_ivar;
 static ID id_deferred_exception_source_id_ivar;
+static ID id_native_safety_signatures_ivar;
 static ID id_socket_io_cache_ivar;
 
 #ifdef RDOC_NEVER_DEFINED
@@ -286,6 +287,33 @@ void rb_curl_multi_forget_easy(ruby_curl_multi *rbcm, void *rbce_ptr) {
   st_delete(rbcm->attached, &key, NULL);
 }
 
+CURLMcode rb_curl_multi_detach_easy(ruby_curl_multi *rbcm, void *rbce_ptr) {
+  ruby_curl_easy *rbce = (ruby_curl_easy *)rbce_ptr;
+  st_data_t key;
+
+  if (!rbcm || !rbce || !rbcm->attached) {
+    return CURLM_OK;
+  }
+
+  key = (st_data_t)rbce;
+  if (!st_delete(rbcm->attached, &key, NULL)) {
+    return CURLM_OK;
+  }
+
+  if (rbcm->handle && rbce->curl) {
+    CURLMcode result = curl_multi_remove_handle(rbcm->handle, rbce->curl);
+    if (result != CURLM_OK) {
+      return result;
+    }
+  }
+
+  if (rbcm->active > 0) {
+    rbcm->active--;
+  }
+
+  return CURLM_OK;
+}
+
 static void rb_curl_multi_detach_all(ruby_curl_multi *rbcm) {
   if (!rbcm || !rbcm->attached) {
     return;
@@ -314,6 +342,8 @@ static int rb_curl_multi_has_easy(ruby_curl_multi *rbcm, ruby_curl_easy *rbce) {
 
 static void rb_curl_multi_remove_request_reference(VALUE self, VALUE easy) {
   VALUE requests;
+  VALUE object_id;
+  VALUE safety_signatures;
 
   if (NIL_P(self) || NIL_P(easy)) {
     return;
@@ -324,7 +354,17 @@ static void rb_curl_multi_remove_request_reference(VALUE self, VALUE easy) {
     return;
   }
 
-  rb_hash_delete(requests, rb_obj_id(easy));
+  object_id = rb_obj_id(easy);
+  rb_hash_delete(requests, object_id);
+
+  if (!rb_ivar_defined(self, id_native_safety_signatures_ivar)) {
+    return;
+  }
+
+  safety_signatures = rb_ivar_get(self, id_native_safety_signatures_ivar);
+  if (RB_TYPE_P(safety_signatures, T_HASH)) {
+    rb_hash_delete(safety_signatures, object_id);
+  }
 }
 
 /* TypedData-compatible free function */
@@ -1427,6 +1467,7 @@ static void rb_curl_multi_socket_drive(VALUE self, ruby_curl_multi *rbcm, multi_
     long wait_ms = cCurlMutiDefaulttimeout;
 
     if (multi_socket_timer_due(ctx)) {
+      ctx->timeout_deadline_ms = -1;
       mrc = curl_multi_socket_action(rbcm->handle, CURL_SOCKET_TIMEOUT, 0, &rbcm->running);
       curb_debugf("[curb.socket] socket_action timeout(due) -> mrc=%d running=%d", mrc, rbcm->running);
       if (mrc != CURLM_OK) raise_curl_multi_error_exception(mrc);
@@ -1688,10 +1729,14 @@ static void rb_curl_multi_socket_drive(VALUE self, ruby_curl_multi *rbcm, multi_
 #else
       rb_thread_wait_for(tv);
 #endif
-      did_timeout = multi_socket_timer_due(ctx);
+      /* libcurl can report active work without a socket callback or deadline;
+       * drive the timeout socket after the scheduler-aware sleep so the state
+       * machine does not stall indefinitely. */
+      did_timeout = 1;
     }
 
     if (did_timeout) {
+      ctx->timeout_deadline_ms = -1;
       mrc = curl_multi_socket_action(rbcm->handle, CURL_SOCKET_TIMEOUT, 0, &rbcm->running);
       curb_debugf("[curb.socket] socket_action timeout -> mrc=%d running=%d", mrc, rbcm->running);
       if (mrc != CURLM_OK) raise_curl_multi_error_exception(mrc);
@@ -2190,6 +2235,7 @@ void init_curb_multi() {
   idCall = rb_intern("call");
   id_deferred_exception_ivar = rb_intern("@__curb_deferred_exception");
   id_deferred_exception_source_id_ivar = rb_intern("@__curb_deferred_exception_source_id");
+  id_native_safety_signatures_ivar = rb_intern("@__curb_native_safety_signatures");
   id_socket_io_cache_ivar = rb_intern("@__curb_socket_io_cache");
   cCurlMulti = rb_define_class_under(mCurl, "Multi", rb_cObject);
 

data/ext/curb_multi.h

@@ -28,6 +28,7 @@ extern const rb_data_type_t ruby_curl_multi_data_type;
 
 void init_curb_multi();
 void rb_curl_multi_forget_easy(ruby_curl_multi *rbcm, void *rbce_ptr);
+CURLMcode rb_curl_multi_detach_easy(ruby_curl_multi *rbcm, void *rbce_ptr);
 
 
 #endif

data/ext/extconf.rb

@@ -298,6 +298,8 @@ have_constant "curlopt_sockoptfunction"
 have_constant "curlopt_sockoptdata"
 have_constant "curlopt_opensocketfunction"
 have_constant "curlopt_opensocketdata"
+have_constant "curlopt_prereqfunction"
+have_constant "curlopt_prereqdata"
 
 # Deprecated constants (still check for them for backward compat)
 have_constant "curlopt_ioctlfunction"
@@ -391,6 +393,7 @@ have_constant "curlopt_socks5_gssapi_nec"
 have_constant "curlopt_interface"
 have_constant "curlopt_localport"
 have_constant "curlopt_dns_cache_timeout"
+have_constant "curlopt_dns_servers"
 have_constant "curlopt_dns_use_global_cache"
 have_constant "curlopt_buffersize"
 have_constant "curlopt_port"
@@ -531,6 +534,7 @@ have_constant "curlusessl_none"
 have_constant "curlusessl_try"
 have_constant "curlusessl_control"
 have_constant "curlusessl_all"
+have_constant "curlopt_connect_to"
 have_constant "curlopt_resolve"
 have_constant "curlopt_request_target"
 have_constant "curlopt_sslcert"
@@ -558,6 +562,10 @@ have_constant :CURL_SSLVERSION_TLSv1_2
 have_constant :CURL_SSLVERSION_TLSv1_3
 
 have_constant "curlopt_ssl_verifypeer"
+have_constant "curlopt_doh_url"
+have_constant "curlopt_doh_ssl_verifypeer"
+have_constant "curlopt_doh_ssl_verifyhost"
+have_constant "curlopt_doh_ssl_verifystatus"
 have_constant "curlopt_cainfo"
 have_constant "curlopt_issuercert"
 have_constant "curlopt_capath"

data/lib/curl/download.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'tempfile'
+
+module Curl
+  class DownloadTargetExistsError < Errno::EEXIST; end
+  DOWNLOAD_OPTION_KEYS = [:download_dir, :overwrite].freeze
+
+  class << self
+    def download_options_hash?(value)
+      value.is_a?(Hash) && value.keys.all? { |key| DOWNLOAD_OPTION_KEYS.include?(key) }
+    end
+
+    def normalize_download_arguments(filename, options)
+      if download_options_hash?(filename) && options.empty?
+        options = filename
+        filename = nil
+      end
+
+      [filename, options || {}]
+    end
+
+    def parse_download_options(options)
+      raise ArgumentError, "download options must be a Hash" unless options.is_a?(Hash)
+
+      options = options.dup
+      download_dir = options.delete(:download_dir)
+      overwrite = options.key?(:overwrite) ? !!options.delete(:overwrite) : false
+      raise ArgumentError, "unsupported download option(s): #{options.keys.join(', ')}" unless options.empty?
+
+      [download_dir, overwrite]
+    end
+
+    def resolve_download_output(url, filename = nil, options = {})
+      filename, options = normalize_download_arguments(filename, options)
+      download_dir, overwrite = parse_download_options(options)
+
+      if filename.is_a?(IO)
+        filename.binmode if filename.respond_to?(:binmode)
+        return [nil, filename, false, overwrite]
+      end
+
+      path = if download_dir
+        safe_download_path(url, download_dir, filename: filename)
+      elsif filename.nil?
+        safe_download_path(url)
+      else
+        filename.to_s
+      end
+
+      [path, nil, true, overwrite]
+    end
+
+    def prepare_download_output(url, filename = nil, options = {})
+      path, io, safe_output, overwrite = resolve_download_output(url, filename, options)
+      return [path, io, safe_output] unless safe_output
+
+      [path, open_safe_download_output(path, overwrite: overwrite), true]
+    end
+
+    def download_filename_from_url(url)
+      path = begin
+        URI.parse(url.to_s).path
+      rescue URI::InvalidURIError
+        url.to_s.split(/\?/, 2).first
+      end
+
+      basename = File.basename(path.to_s)
+      validate_download_filename!(basename)
+    end
+
+    def safe_download_path(url, destination_dir = Dir.pwd, filename: nil)
+      dir = File.expand_path(destination_dir.to_s)
+      raise ArgumentError, "download destination directory does not exist: #{destination_dir}" unless File.directory?(dir)
+
+      File.join(dir, filename.nil? ? download_filename_from_url(url) : validate_download_filename!(filename))
+    end
+
+    def validate_download_filename!(filename)
+      filename = filename.to_s
+      invalid = filename.empty? ||
+                filename == '.' ||
+                filename == '..' ||
+                filename == File::SEPARATOR ||
+                filename.start_with?('.') ||
+                filename.include?("\0") ||
+                filename.include?(File::SEPARATOR) ||
+                filename.include?('\\') ||
+                filename.match?(/\A[A-Za-z]:/) ||
+                (File::ALT_SEPARATOR && filename.include?(File::ALT_SEPARATOR))
+
+      raise ArgumentError, "unsafe download filename derived from URL: #{filename.inspect}" if invalid
+
+      filename
+    end
+
+    def open_safe_download_output(path, overwrite: false)
+      SafeDownloadOutput.new(path, overwrite: overwrite)
+    end
+  end
+
+  class SafeDownloadOutput
+    def initialize(path, overwrite: false)
+      @path = File.expand_path(path.to_s)
+      @overwrite = overwrite
+      raise DownloadTargetExistsError, @path if !@overwrite && File.exist?(@path)
+
+      @tmp = Tempfile.new(['.curb-download-', '.tmp'], File.dirname(@path))
+      @tmp.binmode
+      @closed = false
+    end
+
+    attr_reader :path
+
+    def write(data)
+      @tmp.write(data)
+    end
+
+    def <<(data)
+      write(data)
+    end
+
+    def close(success = false)
+      return if @closed
+
+      @tmp.flush unless @tmp.closed?
+      @tmp.close unless @tmp.closed?
+      install_tmp_file if success
+    ensure
+      @tmp.close! if @tmp
+      @closed = true
+    end
+
+    private
+
+    def install_tmp_file
+      if @overwrite
+        File.rename(@tmp.path, @path)
+      else
+        File.link(@tmp.path, @path)
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    rescue NotImplementedError
+      install_tmp_file_by_copy
+    end
+
+    def install_tmp_file_by_copy
+      flags = File::WRONLY | File::CREAT | File::EXCL
+      flags |= File::BINARY if File.const_defined?(:BINARY)
+
+      File.open(@path, flags) do |dest|
+        File.open(@tmp.path, 'rb') { |src| IO.copy_stream(src, dest) }
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    end
+  end
+end

data/lib/curl/easy.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'curl/download'
 module Curl
   class Easy
     class << self
@@ -83,10 +84,12 @@ module Curl
     
     alias_method :_curb_native_close, :close
     alias_method :_curb_native_multi_set, :multi=
+    alias_method :_curb_native_reset, :reset
     
     def close
       previous_multi = self.multi
       result = _curb_native_close
+      __curb_clear_safety_override!
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       result
     end
@@ -94,12 +97,22 @@ module Curl
     def multi=(multi)
       previous_multi = self.multi
       return multi if previous_multi.equal?(multi)
-    
+
+      if previous_multi && previous_multi.requests[self.object_id]
+        previous_multi.remove(self)
+      end
+
       result = _curb_native_multi_set(multi)
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       multi.__send__(:__register_idle_easy_reference, self) if multi
       result
     end
+
+    def reset
+      result = _curb_native_reset
+      __curb_clear_safety_override!
+      result
+    end
     
     alias post http_post
     alias put http_put
@@ -154,6 +167,84 @@ module Curl
       Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
     end
 
+    def allowed_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_PROTOCOLS_STR', 'CURLOPT_PROTOCOLS', protocols)
+    end
+
+    def allowed_redirect_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_REDIR_PROTOCOLS_STR', 'CURLOPT_REDIR_PROTOCOLS', protocols)
+    end
+
+    def safe_http!
+      __curb_set_safety_override!(
+        :protocols => [:http, :https],
+        :redirect_protocols => [:http, :https]
+      )
+    end
+
+    private
+
+    def __curb_set_safety_override!(options)
+      @__curb_safety_override = (defined?(@__curb_safety_override) && @__curb_safety_override) ? @__curb_safety_override.dup : {}
+      @__curb_safety_override[:protocols] = Array(options[:protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:protocols)
+      @__curb_safety_override[:redirect_protocols] = Array(options[:redirect_protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:redirect_protocols)
+      @__curb_safety_override[:max_body_bytes] = options[:max_body_bytes] if options.key?(:max_body_bytes) && options[:max_body_bytes]
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
+      self
+    end
+
+    def __curb_clear_safety_override!
+      had_override = defined?(@__curb_safety_override) && @__curb_safety_override
+      return unless had_override
+      return if frozen?
+
+      @__curb_safety_override = nil
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+    end
+
+    def __curb_safety_override
+      defined?(@__curb_safety_override) ? @__curb_safety_override : nil
+    end
+
+    def __curb_safety_override_generation
+      defined?(@__curb_safety_override_generation) ? @__curb_safety_override_generation : 0
+    end
+
+    def set_protocol_allowlist(string_option, bitmask_option, protocols)
+      protocol_names = Array(protocols).map { |protocol| protocol.to_s.downcase }
+      raise ArgumentError, "at least one protocol is required" if protocol_names.empty?
+
+      valid_names = %w[
+        dict file ftp ftps gopher gophers http https imap imaps ldap ldaps
+        mqtt pop3 pop3s rtmp rtmpe rtmps rtmpt rtmpte rtmpts rtsp scp sftp
+        smb smbs smtp smtps telnet tftp ws wss
+      ]
+
+      if Curl.const_defined?(string_option)
+        protocol_names.each do |name|
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless valid_names.include?(name)
+        end
+
+        setopt(Curl.const_get(string_option), protocol_names.join(','))
+      elsif Curl.const_defined?(bitmask_option)
+        protocol_pairs = protocol_names.map do |name|
+          const_name = "CURLPROTO_#{name.upcase}"
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless Curl.const_defined?(const_name)
+
+          [name, Curl.const_get(const_name)]
+        end
+
+        setopt(Curl.const_get(bitmask_option), protocol_pairs.inject(0) { |mask, pair| mask | pair.last })
+      else
+        raise NotImplementedError, "protocol allowlists are not supported by this libcurl"
+      end
+
+      protocols
+    end
+
+    public
+
     #
     # call-seq:
     #   easy.perform                                     => true
@@ -163,6 +254,7 @@ module Curl
     # the configured HTTP Verb.
     #
     def perform
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
       self.class.flush_deferred_multi_closes
 
       if Curl.scheduler_active? && self.multi.nil?
@@ -210,6 +302,10 @@ module Curl
         raise callback_error
       end
 
+      if respond_to?(:unsafe_destination_error) && (unsafe_destination_error = self.unsafe_destination_error)
+        raise Curl::Err::UnsafeDestinationError, unsafe_destination_error
+      end
+
       if self.last_result != 0 && self.on_failure.nil?
         err_class, err_summary = Curl::Easy.error(self.last_result)
         err_detail = self.last_error
@@ -641,11 +737,15 @@ module Curl
       end
 
       # call-seq:
-      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   Curl::Easy.download(url, filename = nil, options = {}) { |curl| ... }
       #
       # Stream the specified url (via perform) and save the data directly to the
-      # supplied filename (defaults to the last component of the URL path, which will
-      # usually be the filename most simple urls).
+      # supplied filename. The destination is written through a temporary file and
+      # existing files are not overwritten unless <tt>:overwrite => true</tt> is
+      # passed. When filename is omitted, the destination is safely derived from the
+      # last URL path component in the current directory. Pass
+      # <tt>:download_dir</tt> to treat the filename as a basename inside a trusted
+      # directory and reject absolute, parent-directory, dotfile, and nested names.
       #
       # If a block is supplied, it will be passed the curl instance prior to the
       # perform call.
@@ -658,16 +758,11 @@ module Curl
       # returns a size that differs from the data chunk size - in this case, the
       # offending chunk will *not* be written to the file, the file will be closed,
       # and a Curl::Err::AbortedByCallbackError will be raised.
-      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+      def download(url, filename = nil, download_options = {}, &blk)
         curl = Curl::Easy.new(url, &blk)
+        _download_path, output, safe_output = Curl.prepare_download_output(url, filename, download_options)
 
-        output = if filename.is_a? IO
-          filename.binmode if filename.respond_to?(:binmode)
-          filename
-        else
-          File.open(filename, 'wb')
-        end
-
+        performed = false
         begin
           old_on_body = curl.on_body do |data|
             result = old_on_body ?  old_on_body.call(data) : data.length
@@ -675,8 +770,13 @@ module Curl
             result
           end
           curl.perform
+          performed = true
         ensure
-          output.close rescue IOError
+          if safe_output
+            output.close(performed)
+          else
+            output.close rescue IOError
+          end
         end
 
         return curl