curb 0.8.5 → 1.3.6

data/lib/curl/easy.rb

@@ -1,36 +1,159 @@
+# frozen_string_literal: true
+require 'curl/download'
 module Curl
   class Easy
+    class << self
+      def deferred_multi_close_mutex
+        @deferred_multi_close_mutex ||= Mutex.new
+      end
+
+      def deferred_multi_closes
+        deferred_multi_close_mutex.synchronize do
+          (@deferred_multi_closes ||= []).dup
+        end
+      end
+
+      def release_deferred_multi_close(multi, easy)
+        if easy && multi.requests[easy.object_id]
+          begin
+            multi.remove(easy)
+          rescue StandardError
+            # Deferred cleanup only applies to implicit single-easy multis, so
+            # clear any stale Ruby bookkeeping and continue closing the handle.
+            multi.instance_variable_set(:@requests, {})
+          end
+        else
+          multi.instance_variable_set(:@requests, {})
+        end
+
+        multi.instance_variable_set(:@deferred_close, false)
+        multi._close
+        true
+      rescue StandardError
+        false
+      end
 
+      def defer_multi_close(multi, easy, owner: Thread.current)
+        deferred_multi_close_mutex.synchronize do
+          @deferred_multi_closes ||= []
+          return if @deferred_multi_closes.any? { |entry| entry[:multi].equal?(multi) }
+
+          multi.instance_variable_set(:@deferred_close, true)
+          @deferred_multi_closes << { multi: multi, easy: easy, owner: owner }
+        end
+      end
+
+      def flush_deferred_multi_closes(all_threads: false)
+        pending = deferred_multi_close_mutex.synchronize do
+          @deferred_multi_closes ||= []
+
+          if all_threads
+            @deferred_multi_closes.shift(@deferred_multi_closes.length)
+          else
+            owner = Thread.current
+            remaining = []
+            current = []
+
+            @deferred_multi_closes.each do |entry|
+              if entry[:owner].equal?(owner)
+                current << entry
+              else
+                remaining << entry
+              end
+            end
+
+            @deferred_multi_closes = remaining
+            current
+          end
+        end
+
+        pending.each do |entry|
+          multi = entry[:multi]
+          easy = entry[:easy]
+
+          unless release_deferred_multi_close(multi, easy)
+            defer_multi_close(multi, easy, owner: entry[:owner])
+          end
+        end
+      end
+    end
+
+    at_exit do
+      flush_deferred_multi_closes(all_threads: true)
+    end
+    
+    alias_method :_curb_native_close, :close
+    alias_method :_curb_native_multi_set, :multi=
+    alias_method :_curb_native_reset, :reset
+    
+    def close
+      previous_multi = self.multi
+      result = _curb_native_close
+      __curb_clear_safety_override!
+      previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
+      result
+    end
+    
+    def multi=(multi)
+      previous_multi = self.multi
+      return multi if previous_multi.equal?(multi)
+
+      if previous_multi && previous_multi.requests[self.object_id]
+        previous_multi.remove(self)
+      end
+
+      result = _curb_native_multi_set(multi)
+      previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
+      multi.__send__(:__register_idle_easy_reference, self) if multi
+      result
+    end
+
+    def reset
+      result = _curb_native_reset
+      __curb_clear_safety_override!
+      result
+    end
+    
     alias post http_post
     alias put http_put
     alias body body_str
     alias head header_str
 
+    class Error < StandardError
+      attr_accessor :message, :code
+      def initialize(code, msg)
+        self.message = msg
+        self.code = code
+      end
+    end
+
     #
     # call-seq:
     #   easy.status  => String
     #
     def status
-      parts = self.header_str.split(/\s/)
-      status = []
-      parts.shift
-      while parts.size > 0 && parts.first != ''
-        status << parts.shift
-      end
-      status.join(' ')
+      # Matches the last HTTP Status - following the HTTP protocol specification 'Status-Line = HTTP-Version SP Status-Code SP (Opt:)Reason-Phrase CRLF'
+      statuses = self.header_str.to_s.scan(/HTTP\/\d(\.\d)?\s(\d+\s.*)\r\n/).map {|match| match[1] }
+      statuses.last.strip if statuses.length > 0
     end
 
     #
     # call-seq:
     #   easy.set :sym|Fixnum, value
-    # 
+    #
     # set options on the curl easy handle see http://curl.haxx.se/libcurl/c/curl_easy_setopt.html
     #
     def set(opt,val)
       if opt.is_a?(Symbol)
-        setopt(sym2curl(opt), val)
+        option = sym2curl(opt)
       else
-        setopt(opt.to_i, val)
+        option = opt.to_i
+      end
+
+      begin
+        setopt(option, val)
+      rescue TypeError
+        raise TypeError, "Curb doesn't support setting #{opt} [##{option}] option"
       end
     end
 
@@ -44,6 +167,84 @@ module Curl
       Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
     end
 
+    def allowed_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_PROTOCOLS_STR', 'CURLOPT_PROTOCOLS', protocols)
+    end
+
+    def allowed_redirect_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_REDIR_PROTOCOLS_STR', 'CURLOPT_REDIR_PROTOCOLS', protocols)
+    end
+
+    def safe_http!
+      __curb_set_safety_override!(
+        :protocols => [:http, :https],
+        :redirect_protocols => [:http, :https]
+      )
+    end
+
+    private
+
+    def __curb_set_safety_override!(options)
+      @__curb_safety_override = (defined?(@__curb_safety_override) && @__curb_safety_override) ? @__curb_safety_override.dup : {}
+      @__curb_safety_override[:protocols] = Array(options[:protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:protocols)
+      @__curb_safety_override[:redirect_protocols] = Array(options[:redirect_protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:redirect_protocols)
+      @__curb_safety_override[:max_body_bytes] = options[:max_body_bytes] if options.key?(:max_body_bytes) && options[:max_body_bytes]
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
+      self
+    end
+
+    def __curb_clear_safety_override!
+      had_override = defined?(@__curb_safety_override) && @__curb_safety_override
+      return unless had_override
+      return if frozen?
+
+      @__curb_safety_override = nil
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+    end
+
+    def __curb_safety_override
+      defined?(@__curb_safety_override) ? @__curb_safety_override : nil
+    end
+
+    def __curb_safety_override_generation
+      defined?(@__curb_safety_override_generation) ? @__curb_safety_override_generation : 0
+    end
+
+    def set_protocol_allowlist(string_option, bitmask_option, protocols)
+      protocol_names = Array(protocols).map { |protocol| protocol.to_s.downcase }
+      raise ArgumentError, "at least one protocol is required" if protocol_names.empty?
+
+      valid_names = %w[
+        dict file ftp ftps gopher gophers http https imap imaps ldap ldaps
+        mqtt pop3 pop3s rtmp rtmpe rtmps rtmpt rtmpte rtmpts rtsp scp sftp
+        smb smbs smtp smtps telnet tftp ws wss
+      ]
+
+      if Curl.const_defined?(string_option)
+        protocol_names.each do |name|
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless valid_names.include?(name)
+        end
+
+        setopt(Curl.const_get(string_option), protocol_names.join(','))
+      elsif Curl.const_defined?(bitmask_option)
+        protocol_pairs = protocol_names.map do |name|
+          const_name = "CURLPROTO_#{name.upcase}"
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless Curl.const_defined?(const_name)
+
+          [name, Curl.const_get(const_name)]
+        end
+
+        setopt(Curl.const_get(bitmask_option), protocol_pairs.inject(0) { |mask, pair| mask | pair.last })
+      else
+        raise NotImplementedError, "protocol allowlists are not supported by this libcurl"
+      end
+
+      protocols
+    end
+
+    public
+
     #
     # call-seq:
     #   easy.perform                                     => true
@@ -53,13 +254,62 @@ module Curl
     # the configured HTTP Verb.
     #
     def perform
-      self.multi = Curl::Multi.new if self.multi.nil?
-      self.multi.add self
-      ret = self.multi.perform
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
+      self.class.flush_deferred_multi_closes
+
+      if Curl.scheduler_active? && self.multi.nil?
+        ret = Curl.perform_with_scheduler(self)
+      else
+        multi = self.multi
+        created_multi = multi.nil?
+        raised = false
+
+        if created_multi
+          multi = Curl::Multi.new
+          self.multi = multi
+        end
+
+        begin
+          multi.add(self)
+          ret = multi.perform
+          multi.remove(self) if self.multi == multi
+        rescue Exception
+          raised = true
+          raise
+        ensure
+          if created_multi
+            if raised
+              unless self.class.release_deferred_multi_close(multi, self)
+                self.class.defer_multi_close(multi, self)
+              end
+              self.multi = nil if self.multi == multi
+            elsif Curl::Multi.autoclose
+              multi.__send__(:_autoclose)
+              self.multi = nil if self.multi == multi
+            else
+              self.multi = multi
+            end
+          elsif Curl::Multi.autoclose
+            multi.__send__(:_autoclose)
+            self.multi = nil if self.multi == multi
+          else
+            self.multi = multi
+          end
+        end
+      end
+
+      if (callback_error = _take_callback_error)
+        raise callback_error
+      end
+
+      if respond_to?(:unsafe_destination_error) && (unsafe_destination_error = self.unsafe_destination_error)
+        raise Curl::Err::UnsafeDestinationError, unsafe_destination_error
+      end
 
       if self.last_result != 0 && self.on_failure.nil?
-        error = Curl::Easy.error(self.last_result) 
-        raise error.first
+        err_class, err_summary = Curl::Easy.error(self.last_result)
+        err_detail = self.last_error
+        raise err_class.new([err_summary, err_detail].compact.join(": "))
       end
 
       ret
@@ -67,6 +317,7 @@ module Curl
 
     #
     # call-seq:
+
     #
     # easy = Curl::Easy.new
     # easy.nosignal = true
@@ -83,54 +334,59 @@ module Curl
     #   easy.perform
     #
     def delete=(onoff)
-      set :customrequest, onoff ? 'delete' : nil
+      set :customrequest, onoff ? 'DELETE' : nil
       onoff
     end
-    # 
+    #
     # call-seq:
-    # 
+    #
     #  easy = Curl::Easy.new("url")
-    #  easy.version = Curl::HTTP_1_1
-    #  easy.version = Curl::HTTP_1_0
-    #  easy.version = Curl::HTTP_NONE
-    # 
+    #  easy.version = Curl::HTTP_2_0
+    #  easy.http_version = Curl::HTTP_1_1
+    #  easy.http_version = Curl::HTTP_1_0
+    #  easy.http_version = Curl::HTTP_NONE
+    #
     def version=(http_version)
-      set :http_version, http_version
+      self.http_version = http_version
     end
 
-    # 
+    def version
+      http_version
+    end
+
+    #
     # call-seq:
     #   easy.url = "http://some.url/"                    => "http://some.url/"
-    # 
+    #
     # Set the URL for subsequent calls to +perform+. It is acceptable
     # (and even recommended) to reuse Curl::Easy instances by reassigning
     # the URL between calls to +perform+.
-    # 
+    #
     def url=(u)
       set :url, u
     end
 
-    # 
+    #
     # call-seq:
     #   easy.proxy_url = string                          => string
-    # 
+    #
     # Set the URL of the HTTP proxy to use for subsequent calls to +perform+.
     # The URL should specify the the host name or dotted IP address. To specify
     # port number in this string, append :[port] to the end of the host name.
     # The proxy string may be prefixed with [protocol]:// since any such prefix
     # will be ignored. The proxy's port number may optionally be specified with
     # the separate option proxy_port .
-    # 
+    #
     # When you tell the library to use an HTTP proxy, libcurl will transparently
     # convert operations to HTTP even if you specify an FTP URL etc. This may have
     # an impact on what other features of the library you can use, such as
     # FTP specifics that don't work unless you tunnel through the HTTP proxy. Such
     # tunneling is activated with proxy_tunnel = true.
-    # 
+    #
     # libcurl respects the environment variables *http_proxy*, *ftp_proxy*,
     # *all_proxy* etc, if any of those is set. The proxy_url option does however
     # override any possibly set environment variables.
-    # 
+    #
     # Starting with libcurl 7.14.1, the proxy host string given in environment
     # variables can be specified the exact same way as the proxy can be set with
     # proxy_url, including protocol prefix (http://) and embedded user + password.
@@ -139,99 +395,150 @@ module Curl
       set :proxy, url
     end
 
+    #
+    # call-seq:
+    #   easy.request_target = string                    => string
+    #
+    # Set the request-target used in the HTTP request line (libcurl CURLOPT_REQUEST_TARGET).
+    # Useful for absolute-form request targets (e.g., when speaking to proxies) or
+    # special targets like "*" (OPTIONS *). Requires libcurl with CURLOPT_REQUEST_TARGET support.
+    #
+    def request_target=(value)
+      if Curl.const_defined?(:CURLOPT_REQUEST_TARGET)
+        set :request_target, value
+      else
+        raise NotImplementedError, "CURLOPT_REQUEST_TARGET is not supported by this libcurl"
+      end
+    end
+
     def ssl_verify_host=(value)
       value = 1 if value.class == TrueClass
       value = 0 if value.class == FalseClass
       self.ssl_verify_host_integer=value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.ssl_verify_host?                            => boolean
-    # 
+    #
     # Deprecated: call easy.ssl_verify_host instead
     # can be one of [0,1,2]
-    # 
+    #
     # Determine whether this Curl instance will verify that the server cert
     # is for the server it is known as.
-    # 
+    #
     def ssl_verify_host?
       ssl_verify_host.nil? ? false : (ssl_verify_host > 0)
     end
 
-    # 
+    #
     # call-seq:
     #   easy.interface = string                          => string
-    # 
+    #
     # Set the interface name to use as the outgoing network interface.
     # The name can be an interface name, an IP address or a host name.
-    # 
+    #
     def interface=(value)
       set :interface, value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.userpwd = string                            => string
-    # 
+    #
     # Set the username/password string to use for subsequent calls to +perform+.
     # The supplied string should have the form "username:password"
-    # 
+    #
     def userpwd=(value)
       set :userpwd, value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.proxypwd = string                           => string
-    # 
+    #
     # Set the username/password string to use for proxy connection during
     # subsequent calls to +perform+. The supplied string should have the
     # form "username:password"
-    # 
+    #
     def proxypwd=(value)
       set :proxyuserpwd, value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.cookies = "name1=content1; name2=content2;" => string
-    # 
-    # Set cookies to be sent by this Curl::Easy instance. The format of the string should
-    # be NAME=CONTENTS, where NAME is the cookie name and CONTENTS is what the cookie should contain.
-    # Set multiple cookies in one string like this: "name1=content1; name2=content2;" etc.
-    # 
+    #
+    # Set the manual Cookie request header for this Curl::Easy instance.
+    # The format of the string should be NAME=CONTENTS, where NAME is the cookie name and
+    # CONTENTS is what the cookie should contain. Set multiple cookies in one string like this:
+    # "name1=content1; name2=content2;".
+    #
+    # Notes:
+    # - This only affects the outgoing Cookie header (libcurl CURLOPT_COOKIE) and does NOT
+    #   alter the internal libcurl cookie engine (which stores cookies from Set-Cookie).
+    # - To change cookies stored in the engine, use {#cookielist} / {#cookielist=} or
+    #   {#set} with :cookielist.
+    # - To clear a previously set manual Cookie header, assign an empty string ('').
+    #   Assigning +nil+ has no effect in current curb versions.
+    #
     def cookies=(value)
       set :cookie, value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.cookiefile = string                         => string
-    # 
+    #
     # Set a file that contains cookies to be sent in subsequent requests by this Curl::Easy instance.
-    # 
+    #
     # *Note* that you must set enable_cookies true to enable the cookie
     # engine, or this option will be ignored.
-    # 
+    #
+    # Note: assigning +nil+ has no effect; pass a path string to use a cookie file.
+    #
     def cookiefile=(value)
       set :cookiefile, value
     end
 
-    # 
+    #
     # call-seq:
     #   easy.cookiejar = string                          => string
-    # 
-    # Set a cookiejar file to use for this Curl::Easy instance.
-    # Cookies from the response will be written into this file.
-    # 
+    #
+    # Set a cookiejar file to use for this Curl::Easy instance. Cookies from the response
+    # will be written into this file.
+    #
     # *Note* that you must set enable_cookies true to enable the cookie
     # engine, or this option will be ignored.
-    # 
+    #
+    # Note: assigning +nil+ has no effect; pass a path string to persist cookies to a file.
+    #
     def cookiejar=(value)
       set :cookiejar, value
     end
 
+    #
+    # call-seq:
+    #   easy.cookielist = string                         => string
+    #
+    # Modify cookies in libcurl's internal cookie engine (CURLOPT_COOKIELIST).
+    # Accepts a Set-Cookie style string, one or more lines in Netscape cookie file format,
+    # or one of the special commands: "ALL" (clear), "SESS" (remove session cookies),
+    # "FLUSH" (write to jar), "RELOAD" (reload from file).
+    #
+    # Examples:
+    #   easy.cookielist = "Set-Cookie: session=42; Domain=example.com; Path=/;"
+    #   easy.cookielist = [
+    #     ['.example.com', 'TRUE', '/', 'FALSE', 0, 'c1', 'v1'].join("\t"),
+    #     ['.example.com', 'TRUE', '/', 'FALSE', 0, 'c2', 'v2'].join("\t"),
+    #     ''
+    #   ].join("\n")
+    #   easy.cookielist = 'ALL'   # clear all cookies in the engine
+    #
+    def cookielist=(value)
+      set :cookielist, value
+    end
+
     #
     # call-seq:
     #  easy = Curl::Easy.new("url") do|c|
@@ -302,7 +609,7 @@ module Curl
 
       #
       # call-seq:
-      #   Curl::Easy.perform(url) { |easy| ... }           => #&lt;Curl::Easy...&gt;
+      #   Curl::Easy.perform(url) { |easy| ... }           => #<Curl::Easy...>
       #
       # Convenience method that creates a new Curl::Easy instance with
       # the specified URL and calls the general +perform+ method, before returning
@@ -320,7 +627,7 @@ module Curl
 
       #
       # call-seq:
-      #   Curl::Easy.http_get(url) { |easy| ... }          => #&lt;Curl::Easy...&gt;
+      #   Curl::Easy.http_get(url) { |easy| ... }          => #<Curl::Easy...>
       #
       # Convenience method that creates a new Curl::Easy instance with
       # the specified URL and calls +http_get+, before returning the new instance.
@@ -337,7 +644,7 @@ module Curl
 
       #
       # call-seq:
-      #   Curl::Easy.http_head(url) { |easy| ... }         => #&lt;Curl::Easy...&gt;
+      #   Curl::Easy.http_head(url) { |easy| ... }         => #<Curl::Easy...>
       #
       # Convenience method that creates a new Curl::Easy instance with
       # the specified URL and calls +http_head+, before returning the new instance.
@@ -355,13 +662,36 @@ module Curl
       #
       # call-seq:
       #   Curl::Easy.http_put(url, data) {|c| ... }
+      #   Curl::Easy.http_put(url, "some=urlencoded%20form%20data&and=so%20on") => true
+      #   Curl::Easy.http_put(url, "some=urlencoded%20form%20data", "and=so%20on", ...) => true
+      #   Curl::Easy.http_put(url, "some=urlencoded%20form%20data", Curl::PostField, "and=so%20on", ...) => true
+      #   Curl::Easy.http_put(url, Curl::PostField, Curl::PostField ..., Curl::PostField) => true
       #
       # see easy.http_put
       #
-      def http_put(url, data)
-        c = Curl::Easy.new url
+      def http_put(*args)
+        url = args.shift
+        c = Curl::Easy.new(url)
+        yield c if block_given?
+        c.http_put(*args)
+        c
+      end
+
+      #
+      # call-seq:
+      #   Curl::Easy.http_patch(url, data) {|c| ... }
+      #   Curl::Easy.http_patch(url, "some=urlencoded%20form%20data&and=so%20on") => true
+      #   Curl::Easy.http_patch(url, "some=urlencoded%20form%20data", "and=so%20on", ...) => true
+      #   Curl::Easy.http_patch(url, "some=urlencoded%20form%20data", Curl::PostField, "and=so%20on", ...) => true
+      #   Curl::Easy.http_patch(url, Curl::PostField, Curl::PostField ..., Curl::PostField) => true
+      #
+      # see easy.http_patch
+      #
+      def http_patch(*args)
+        url = args.shift
+        c = Curl::Easy.new(url)
         yield c if block_given?
-        c.http_put data
+        c.http_patch(*args)
         c
       end
 
@@ -391,7 +721,7 @@ module Curl
 
       #
       # call-seq:
-      #   Curl::Easy.http_delete(url) { |easy| ... }       => #&lt;Curl::Easy...&gt;
+      #   Curl::Easy.http_delete(url) { |easy| ... }       => #<Curl::Easy...>
       #
       # Convenience method that creates a new Curl::Easy instance with
       # the specified URL and calls +http_delete+, before returning the new instance.
@@ -405,35 +735,34 @@ module Curl
         c.http_delete
         c
       end
-    
+
       # call-seq:
-      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
-      #   
+      #   Curl::Easy.download(url, filename = nil, options = {}) { |curl| ... }
+      #
       # Stream the specified url (via perform) and save the data directly to the
-      # supplied filename (defaults to the last component of the URL path, which will 
-      # usually be the filename most simple urls).
-      # 
+      # supplied filename. The destination is written through a temporary file and
+      # existing files are not overwritten unless <tt>:overwrite => true</tt> is
+      # passed. When filename is omitted, the destination is safely derived from the
+      # last URL path component in the current directory. Pass
+      # <tt>:download_dir</tt> to treat the filename as a basename inside a trusted
+      # directory and reject absolute, parent-directory, dotfile, and nested names.
+      #
       # If a block is supplied, it will be passed the curl instance prior to the
       # perform call.
-      # 
+      #
       # *Note* that the semantics of the on_body handler are subtly changed when using
-      # download, to account for the automatic routing of data to the specified file: The 
+      # download, to account for the automatic routing of data to the specified file: The
       # data string is passed to the handler *before* it is written
-      # to the file, allowing the handler to perform mutative operations where 
+      # to the file, allowing the handler to perform mutative operations where
       # necessary. As usual, the transfer will be aborted if the on_body handler
-      # returns a size that differs from the data chunk size - in this case, the 
+      # returns a size that differs from the data chunk size - in this case, the
       # offending chunk will *not* be written to the file, the file will be closed,
       # and a Curl::Err::AbortedByCallbackError will be raised.
-      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+      def download(url, filename = nil, download_options = {}, &blk)
         curl = Curl::Easy.new(url, &blk)
-        
-        output = if filename.is_a? IO
-          filename.binmode if filename.respond_to?(:binmode)
-          filename
-        else
-          File.open(filename, 'wb')
-        end
-        
+        _download_path, output, safe_output = Curl.prepare_download_output(url, filename, download_options)
+
+        performed = false
         begin
           old_on_body = curl.on_body do |data|
             result = old_on_body ?  old_on_body.call(data) : data.length
@@ -441,10 +770,15 @@ module Curl
             result
           end
           curl.perform
+          performed = true
         ensure
-          output.close rescue IOError
+          if safe_output
+            output.close(performed)
+          else
+            output.close rescue IOError
+          end
         end
-        
+
         return curl
       end
     end