curb 1.3.4 → 1.3.6

data/lib/curl/download.rb

@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+require 'uri'
+require 'tempfile'
+
+module Curl
+  class DownloadTargetExistsError < Errno::EEXIST; end
+  DOWNLOAD_OPTION_KEYS = [:download_dir, :overwrite].freeze
+
+  class << self
+    def download_options_hash?(value)
+      value.is_a?(Hash) && value.keys.all? { |key| DOWNLOAD_OPTION_KEYS.include?(key) }
+    end
+
+    def normalize_download_arguments(filename, options)
+      if download_options_hash?(filename) && options.empty?
+        options = filename
+        filename = nil
+      end
+
+      [filename, options || {}]
+    end
+
+    def parse_download_options(options)
+      raise ArgumentError, "download options must be a Hash" unless options.is_a?(Hash)
+
+      options = options.dup
+      download_dir = options.delete(:download_dir)
+      overwrite = options.key?(:overwrite) ? !!options.delete(:overwrite) : false
+      raise ArgumentError, "unsupported download option(s): #{options.keys.join(', ')}" unless options.empty?
+
+      [download_dir, overwrite]
+    end
+
+    def resolve_download_output(url, filename = nil, options = {})
+      filename, options = normalize_download_arguments(filename, options)
+      download_dir, overwrite = parse_download_options(options)
+
+      if filename.is_a?(IO)
+        filename.binmode if filename.respond_to?(:binmode)
+        return [nil, filename, false, overwrite]
+      end
+
+      path = if download_dir
+        safe_download_path(url, download_dir, filename: filename)
+      elsif filename.nil?
+        safe_download_path(url)
+      else
+        filename.to_s
+      end
+
+      [path, nil, true, overwrite]
+    end
+
+    def prepare_download_output(url, filename = nil, options = {})
+      path, io, safe_output, overwrite = resolve_download_output(url, filename, options)
+      return [path, io, safe_output] unless safe_output
+
+      [path, open_safe_download_output(path, overwrite: overwrite), true]
+    end
+
+    def download_filename_from_url(url)
+      path = begin
+        URI.parse(url.to_s).path
+      rescue URI::InvalidURIError
+        url.to_s.split(/\?/, 2).first
+      end
+
+      basename = File.basename(path.to_s)
+      validate_download_filename!(basename)
+    end
+
+    def safe_download_path(url, destination_dir = Dir.pwd, filename: nil)
+      dir = File.expand_path(destination_dir.to_s)
+      raise ArgumentError, "download destination directory does not exist: #{destination_dir}" unless File.directory?(dir)
+
+      File.join(dir, filename.nil? ? download_filename_from_url(url) : validate_download_filename!(filename))
+    end
+
+    def validate_download_filename!(filename)
+      filename = filename.to_s
+      invalid = filename.empty? ||
+                filename == '.' ||
+                filename == '..' ||
+                filename == File::SEPARATOR ||
+                filename.start_with?('.') ||
+                filename.include?("\0") ||
+                filename.include?(File::SEPARATOR) ||
+                filename.include?('\\') ||
+                filename.match?(/\A[A-Za-z]:/) ||
+                (File::ALT_SEPARATOR && filename.include?(File::ALT_SEPARATOR))
+
+      raise ArgumentError, "unsafe download filename derived from URL: #{filename.inspect}" if invalid
+
+      filename
+    end
+
+    def open_safe_download_output(path, overwrite: false)
+      SafeDownloadOutput.new(path, overwrite: overwrite)
+    end
+  end
+
+  class SafeDownloadOutput
+    def initialize(path, overwrite: false)
+      @path = File.expand_path(path.to_s)
+      @overwrite = overwrite
+      raise DownloadTargetExistsError, @path if !@overwrite && File.exist?(@path)
+
+      @tmp = Tempfile.new(['.curb-download-', '.tmp'], File.dirname(@path))
+      @tmp.binmode
+      @closed = false
+    end
+
+    attr_reader :path
+
+    def write(data)
+      @tmp.write(data)
+    end
+
+    def <<(data)
+      write(data)
+    end
+
+    def close(success = false)
+      return if @closed
+
+      @tmp.flush unless @tmp.closed?
+      @tmp.close unless @tmp.closed?
+      install_tmp_file if success
+    ensure
+      @tmp.close! if @tmp
+      @closed = true
+    end
+
+    private
+
+    def install_tmp_file
+      if @overwrite
+        File.rename(@tmp.path, @path)
+      else
+        File.link(@tmp.path, @path)
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    rescue NotImplementedError
+      install_tmp_file_by_copy
+    end
+
+    def install_tmp_file_by_copy
+      flags = File::WRONLY | File::CREAT | File::EXCL
+      flags |= File::BINARY if File.const_defined?(:BINARY)
+
+      File.open(@path, flags) do |dest|
+        File.open(@tmp.path, 'rb') { |src| IO.copy_stream(src, dest) }
+      end
+    rescue Errno::EEXIST
+      raise DownloadTargetExistsError, @path
+    end
+  end
+end

data/lib/curl/easy.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+require 'curl/download'
 module Curl
   class Easy
     class << self
@@ -83,10 +84,12 @@ module Curl
     
     alias_method :_curb_native_close, :close
     alias_method :_curb_native_multi_set, :multi=
+    alias_method :_curb_native_reset, :reset
     
     def close
       previous_multi = self.multi
       result = _curb_native_close
+      __curb_clear_safety_override!
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       result
     end
@@ -94,12 +97,22 @@ module Curl
     def multi=(multi)
       previous_multi = self.multi
       return multi if previous_multi.equal?(multi)
-    
+
+      if previous_multi && previous_multi.requests[self.object_id]
+        previous_multi.remove(self)
+      end
+
       result = _curb_native_multi_set(multi)
       previous_multi.__send__(:__unregister_idle_easy_reference, self) if previous_multi
       multi.__send__(:__register_idle_easy_reference, self) if multi
       result
     end
+
+    def reset
+      result = _curb_native_reset
+      __curb_clear_safety_override!
+      result
+    end
     
     alias post http_post
     alias put http_put
@@ -154,6 +167,84 @@ module Curl
       Curl.const_get("CURLOPT_#{opt.to_s.upcase}")
     end
 
+    def allowed_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_PROTOCOLS_STR', 'CURLOPT_PROTOCOLS', protocols)
+    end
+
+    def allowed_redirect_protocols=(protocols)
+      set_protocol_allowlist('CURLOPT_REDIR_PROTOCOLS_STR', 'CURLOPT_REDIR_PROTOCOLS', protocols)
+    end
+
+    def safe_http!
+      __curb_set_safety_override!(
+        :protocols => [:http, :https],
+        :redirect_protocols => [:http, :https]
+      )
+    end
+
+    private
+
+    def __curb_set_safety_override!(options)
+      @__curb_safety_override = (defined?(@__curb_safety_override) && @__curb_safety_override) ? @__curb_safety_override.dup : {}
+      @__curb_safety_override[:protocols] = Array(options[:protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:protocols)
+      @__curb_safety_override[:redirect_protocols] = Array(options[:redirect_protocols]).map { |protocol| protocol.to_s.downcase.to_sym } if options.key?(:redirect_protocols)
+      @__curb_safety_override[:max_body_bytes] = options[:max_body_bytes] if options.key?(:max_body_bytes) && options[:max_body_bytes]
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
+      self
+    end
+
+    def __curb_clear_safety_override!
+      had_override = defined?(@__curb_safety_override) && @__curb_safety_override
+      return unless had_override
+      return if frozen?
+
+      @__curb_safety_override = nil
+      @__curb_safety_override_generation = __curb_safety_override_generation + 1
+    end
+
+    def __curb_safety_override
+      defined?(@__curb_safety_override) ? @__curb_safety_override : nil
+    end
+
+    def __curb_safety_override_generation
+      defined?(@__curb_safety_override_generation) ? @__curb_safety_override_generation : 0
+    end
+
+    def set_protocol_allowlist(string_option, bitmask_option, protocols)
+      protocol_names = Array(protocols).map { |protocol| protocol.to_s.downcase }
+      raise ArgumentError, "at least one protocol is required" if protocol_names.empty?
+
+      valid_names = %w[
+        dict file ftp ftps gopher gophers http https imap imaps ldap ldaps
+        mqtt pop3 pop3s rtmp rtmpe rtmps rtmpt rtmpte rtmpts rtsp scp sftp
+        smb smbs smtp smtps telnet tftp ws wss
+      ]
+
+      if Curl.const_defined?(string_option)
+        protocol_names.each do |name|
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless valid_names.include?(name)
+        end
+
+        setopt(Curl.const_get(string_option), protocol_names.join(','))
+      elsif Curl.const_defined?(bitmask_option)
+        protocol_pairs = protocol_names.map do |name|
+          const_name = "CURLPROTO_#{name.upcase}"
+          raise ArgumentError, "unsupported protocol: #{name.inspect}" unless Curl.const_defined?(const_name)
+
+          [name, Curl.const_get(const_name)]
+        end
+
+        setopt(Curl.const_get(bitmask_option), protocol_pairs.inject(0) { |mask, pair| mask | pair.last })
+      else
+        raise NotImplementedError, "protocol allowlists are not supported by this libcurl"
+      end
+
+      protocols
+    end
+
+    public
+
     #
     # call-seq:
     #   easy.perform                                     => true
@@ -163,6 +254,7 @@ module Curl
     # the configured HTTP Verb.
     #
     def perform
+      Curl.__send__(:apply_safety!, self) if Curl.respond_to?(:apply_safety!, true)
       self.class.flush_deferred_multi_closes
 
       if Curl.scheduler_active? && self.multi.nil?
@@ -210,6 +302,10 @@ module Curl
         raise callback_error
       end
 
+      if respond_to?(:unsafe_destination_error) && (unsafe_destination_error = self.unsafe_destination_error)
+        raise Curl::Err::UnsafeDestinationError, unsafe_destination_error
+      end
+
       if self.last_result != 0 && self.on_failure.nil?
         err_class, err_summary = Curl::Easy.error(self.last_result)
         err_detail = self.last_error
@@ -641,11 +737,15 @@ module Curl
       end
 
       # call-seq:
-      #   Curl::Easy.download(url, filename = url.split(/\?/).first.split(/\//).last) { |curl| ... }
+      #   Curl::Easy.download(url, filename = nil, options = {}) { |curl| ... }
       #
       # Stream the specified url (via perform) and save the data directly to the
-      # supplied filename (defaults to the last component of the URL path, which will
-      # usually be the filename most simple urls).
+      # supplied filename. The destination is written through a temporary file and
+      # existing files are not overwritten unless <tt>:overwrite => true</tt> is
+      # passed. When filename is omitted, the destination is safely derived from the
+      # last URL path component in the current directory. Pass
+      # <tt>:download_dir</tt> to treat the filename as a basename inside a trusted
+      # directory and reject absolute, parent-directory, dotfile, and nested names.
       #
       # If a block is supplied, it will be passed the curl instance prior to the
       # perform call.
@@ -658,16 +758,11 @@ module Curl
       # returns a size that differs from the data chunk size - in this case, the
       # offending chunk will *not* be written to the file, the file will be closed,
       # and a Curl::Err::AbortedByCallbackError will be raised.
-      def download(url, filename = url.split(/\?/).first.split(/\//).last, &blk)
+      def download(url, filename = nil, download_options = {}, &blk)
         curl = Curl::Easy.new(url, &blk)
+        _download_path, output, safe_output = Curl.prepare_download_output(url, filename, download_options)
 
-        output = if filename.is_a? IO
-          filename.binmode if filename.respond_to?(:binmode)
-          filename
-        else
-          File.open(filename, 'wb')
-        end
-
+        performed = false
         begin
           old_on_body = curl.on_body do |data|
             result = old_on_body ?  old_on_body.call(data) : data.length
@@ -675,8 +770,13 @@ module Curl
             result
           end
           curl.perform
+          performed = true
         ensure
-          output.close rescue IOError
+          if safe_output
+            output.close(performed)
+          else
+            output.close rescue IOError
+          end
         end
 
         return curl

data/lib/curl/multi.rb

@@ -1,9 +1,19 @@
 # frozen_string_literal: true
+require 'curl/download'
 module Curl
   class Multi
     class DownloadError < RuntimeError
       attr_accessor :errors
     end
+
+    IDLE_EASY_REFERENCES_USE_WEAK_MAP = begin
+      probe = ObjectSpace::WeakMap.new
+      probe[Object.new.freeze] = true
+      true
+    rescue ArgumentError, FrozenError, NameError
+      false
+    end
+
     class << self
       # call-seq:
       #   Curl::Multi.get(['url1','url2','url3','url4','url5'], :follow_location => true) do|easy|
@@ -102,6 +112,7 @@ module Curl
           url     = c.delete(:url)
           method  = c.delete(:method)
           headers = c.delete(:headers)
+          internal_info = c.delete(:__curb_internal_info)
 
           easy    = Curl::Easy.new if easy.nil?
 
@@ -140,7 +151,13 @@ module Curl
 
           easy.on_complete {|curl|
             free_handles << curl
-            blk.call(curl,curl.response_code,method) if blk
+            if blk
+              if internal_info
+                blk.call(curl,curl.response_code,method,internal_info)
+              else
+                blk.call(curl,curl.response_code,method)
+              end
+            end
           }
           m.add(easy)
         end
@@ -182,73 +199,122 @@ module Curl
       #
       # Curl::Multi.download(['http://example.com/p/a/t/h/file1.txt','http://example.com/p/a/t/h/file2.txt']){|c|}
       #
-      # will create 2 new files file1.txt and file2.txt
+      # will create 2 new files file1.txt and file2.txt, unless either file
+      # already exists. Auto-derived filenames are safely derived from the last
+      # URL path component. Pass <tt>:download_dir</tt> as the fifth argument to
+      # treat download_paths as basenames inside a trusted directory and reject
+      # absolute, parent-directory, dotfile, and nested names.
       #
       # 2 files will be opened, and remain open until the call completes
       #
       # when using the :post or :put method, urls should be a hash, including the individual post fields per post
       #
-      def download(urls,easy_options={},multi_options={},download_paths=nil,&blk)
+      def download(urls,easy_options={},multi_options={},download_paths=nil,download_options={},&blk)
         errors = []
         procs = []
         files = []
         urls_with_config = []
-        url_to_download_paths = {}
+        seen_download_paths = {}
+        download_infos = []
+
+        if Curl.download_options_hash?(download_paths) && download_options.empty?
+          download_options = download_paths
+          download_paths = nil
+        end
 
         urls.each_with_index do|urlcfg,i|
           if urlcfg.is_a?(Hash)
-            url = url[:url]
+            url = urlcfg[:url]
           else
             url = urlcfg
           end
 
-          if download_paths and download_paths[i]
-            download_path = download_paths[i]
-          else
-            download_path = File.basename(url)
+          download_path_arg = download_paths && download_paths[i]
+          download_path, file, safe_output, overwrite = Curl.resolve_download_output(url, download_path_arg, download_options)
+
+          if safe_output
+            expanded_path = File.expand_path(download_path)
+            raise ArgumentError, "duplicate download destination: #{download_path}" if seen_download_paths[expanded_path]
+
+            seen_download_paths[expanded_path] = true
           end
 
-          file = lambda do|dp|
-            file = File.open(dp,"wb")
-            procs << (lambda {|data| file.write data; data.size })
-            files << file
-            file
-          end.call(download_path)
+          download_infos << {
+            :url => url,
+            :urlcfg => urlcfg,
+            :path => download_path,
+            :file => file,
+            :safe_output => safe_output,
+            :overwrite => overwrite
+          }
+        end
 
-          if urlcfg.is_a?(Hash)
-            urls_with_config << urlcfg.merge({:on_body => procs.last}.merge(easy_options))
+        download_infos.each do |info|
+          info[:file] ||= Curl.open_safe_download_output(info[:path], :overwrite => info[:overwrite])
+          file = info[:file]
+          procs << (lambda {|data| file.write data; data.size })
+          files << file
+
+          if info[:urlcfg].is_a?(Hash)
+            urls_with_config << info[:urlcfg].merge({:on_body => procs.last, :__curb_internal_info => info}.merge(easy_options))
           else
-            urls_with_config << {:url => url, :on_body => procs.last, :method => :get}.merge(easy_options)
+            urls_with_config << {:url => info[:url], :on_body => procs.last, :method => :get, :__curb_internal_info => info}.merge(easy_options)
           end
-          url_to_download_paths[url] = {:path => download_path, :file => file} # store for later
         end
 
-        if blk
-          # when injecting the block, ensure file is closed before yielding
-          Curl::Multi.http(urls_with_config, multi_options) do |c,code,method|
-            info = url_to_download_paths[c.url]
+        finalize_download = lambda do |curl, info|
+          file = info[:file]
+          files.reject!{|f| f == file }
+
+          if curl.last_result != 0
             begin
-              file = info[:file]
-              files.reject!{|f| f == file }
-              file.close
+              if info[:safe_output]
+                file.close(false)
+              else
+                file.close
+              end
             rescue => e
               errors << e
             end
+            err_class, err_summary = Curl::Easy.error(curl.last_result)
+            err_detail = curl.last_error
+            errors << err_class.new([err_summary, err_detail].compact.join(": "))
+            false
+          else
+            begin
+              if info[:safe_output]
+                file.close(true)
+              else
+                file.close
+              end
+              true
+            rescue => e
+              errors << e
+              false
+            end
+          end
+        end
+
+        Curl::Multi.http(urls_with_config, multi_options) do |c,code,method,info|
+          if finalize_download.call(c, info) && blk
             blk.call(c,info[:path])
           end
-        else
-          Curl::Multi.http(urls_with_config, multi_options)
         end
 
       ensure
+        pending_exception = $!
         files.each {|f|
           begin
-            f.close
+            if f.is_a?(Curl::SafeDownloadOutput)
+              f.close(false)
+            else
+              f.close
+            end
           rescue => e
             errors << e
           end
         }
-        if errors.any?
+        if errors.any? && !pending_exception
           de = Curl::Multi::DownloadError.new
           de.errors = errors
           raise de
@@ -270,19 +336,46 @@ module Curl
       @requests ||= {}
     end
 
+    def __curb_native_safety_signatures
+      @__curb_native_safety_signatures ||= {}
+    end
+
+    def __curb_safety_signature_for(easy)
+      safety_signature = if Curl.respond_to?(:safety_signature_for, true)
+        Curl.__send__(:safety_signature_for, easy)
+      end
+
+      [
+        safety_signature,
+        easy.respond_to?(:network_policy) ? easy.network_policy : nil,
+        easy.respond_to?(:allowed_hosts) ? easy.allowed_hosts : nil,
+        easy.respond_to?(:allowed_cidrs) ? easy.allowed_cidrs : nil
+      ]
+    end
+
+    def __record_native_safety_signature(easy)
+      __curb_native_safety_signatures[easy.object_id] = __curb_safety_signature_for(easy)
+    end
+
     def __idle_easy_references
-      @__curb_idle_easy_references ||= ObjectSpace::WeakMap.new
+      @__curb_idle_easy_references ||= __new_idle_easy_references
     end
     
     def __register_idle_easy_reference(easy)
-      __idle_easy_references[easy] = true
+      if IDLE_EASY_REFERENCES_USE_WEAK_MAP
+        __idle_easy_references[easy] = true
+      else
+        __idle_easy_references[easy.object_id] = true
+      end
       self
     end
     
     def __unregister_idle_easy_reference(easy)
       return self unless instance_variable_defined?(:@__curb_idle_easy_references)
 
-      if @__curb_idle_easy_references.respond_to?(:delete)
+      if !IDLE_EASY_REFERENCES_USE_WEAK_MAP
+        @__curb_idle_easy_references.delete(easy.object_id)
+      elsif @__curb_idle_easy_references.respond_to?(:delete)
         @__curb_idle_easy_references.delete(easy)
       else
         retained_references = ObjectSpace::WeakMap.new
@@ -299,30 +392,70 @@ module Curl
     
     def __clear_idle_easy_references
       return unless instance_variable_defined?(:@__curb_idle_easy_references)
-    
-      @__curb_idle_easy_references.keys.each do |easy|
-        easy.multi = nil if easy.multi.equal?(self)
+
+      if IDLE_EASY_REFERENCES_USE_WEAK_MAP
+        @__curb_idle_easy_references.keys.each do |easy|
+          easy.multi = nil if easy.multi.equal?(self)
+        end
+      else
+        @__curb_idle_easy_references.keys.each do |easy_object_id|
+          begin
+            easy = ObjectSpace._id2ref(easy_object_id)
+          rescue RangeError
+            next
+          end
+
+          next unless easy.is_a?(Curl::Easy)
+
+          easy.multi = nil if easy.multi.equal?(self)
+        end
       end
-      @__curb_idle_easy_references = ObjectSpace::WeakMap.new
+      @__curb_idle_easy_references = __new_idle_easy_references
     end
 
-    private :__idle_easy_references, :__register_idle_easy_reference,
-            :__unregister_idle_easy_reference, :__clear_idle_easy_references
+    def __new_idle_easy_references
+      IDLE_EASY_REFERENCES_USE_WEAK_MAP ? ObjectSpace::WeakMap.new : {}
+    end
+
+    private :__idle_easy_references, :__curb_native_safety_signatures,
+            :__curb_safety_signature_for, :__record_native_safety_signature,
+            :__register_idle_easy_reference,
+            :__unregister_idle_easy_reference, :__clear_idle_easy_references,
+            :__new_idle_easy_references
+
+    alias_method :_curb_native_perform, :perform
+
+    def perform(*args, &block)
+      requests.each_value do |easy|
+        Curl.__send__(:apply_safety!, easy) if Curl.respond_to?(:apply_safety!, true)
+        signature = __curb_safety_signature_for(easy)
+        if __curb_native_safety_signatures[easy.object_id] != signature &&
+           easy.respond_to?(:__curb_native_setup!, true)
+          easy.__send__(:__curb_native_setup!)
+          __curb_native_safety_signatures[easy.object_id] = signature
+        end
+      end
+
+      _curb_native_perform(*args, &block)
+    end
 
     def add(easy)
       return self if requests[easy.object_id]
       # Once a deferred callback exception is pending, Multi#perform is
       # draining existing transfers only and must not start replacement work.
       return self if instance_variable_defined?(:@__curb_deferred_exception)
+      Curl.__send__(:apply_safety!, easy) if Curl.respond_to?(:apply_safety!, true)
       _add(easy)
       __unregister_idle_easy_reference(easy)
       requests[easy.object_id] = easy
+      __record_native_safety_signature(easy)
       self
     end
 
     def remove(easy)
       return self if !requests[easy.object_id]
       requests.delete(easy.object_id)
+      __curb_native_safety_signatures.delete(easy.object_id)
       _remove(easy)
       self
     end