cloudinary 1.29.0 → 2.0.0

data/lib/cloudinary/migrator.rb

@@ -6,29 +6,29 @@ class Cloudinary::Migrator
   attr_reader :db
   attr_reader :work, :results, :mutex
   attr_reader :extra_options
-  
-  
+
+
   @@init = false
   def self.init
     return if @@init
     @@init = true
 
-    begin 
+    begin
       require 'sqlite3'
     rescue LoadError
-      raise "Please add sqlite3 to your Gemfile"    
+      raise "Please add sqlite3 to your Gemfile"
     end
     require 'tempfile'
   end
-  
+
   def json_decode(str)
     Cloudinary::Utils.json_decode(str)
   end
-    
+
   def initialize(options={})
     self.class.init
-    
-    options[:db_file] = "tmp/migration#{$$}.db" if options[:private_database] && !options[:db_file] 
+
+    options[:db_file] = "tmp/migration#{$$}.db" if options[:private_database] && !options[:db_file]
     @dbfile = options[:db_file] || "tmp/migration.db"
     FileUtils.mkdir_p(File.dirname(@dbfile))
     @db = SQLite3::Database.new @dbfile, :results_as_hash=>true
@@ -37,7 +37,6 @@ class Cloudinary::Migrator
     @debug = options[:debug] || false
     @ignore_duplicates = options[:ignore_duplicates]
     @threads = [options[:threads] || 10, 100].min
-    @threads = 1 if RUBY_VERSION < "1.9"
     @extra_options = {:api_key=>options[:api_key], :api_secret=>options[:api_secret]}
     @delete_after_done = options[:delete_after_done] || options[:private_database]
     @max_processing = @threads * 10
@@ -77,7 +76,7 @@ class Cloudinary::Migrator
       @db.execute("delete from queue")
     end
   end
-    
+
   def register_retrieve(&block)
     @retrieve = block
   end
@@ -85,26 +84,26 @@ class Cloudinary::Migrator
   def register_complete(&block)
     @complete = block
   end
-  
-  def process(options={})    
+
+  def process(options={})
     raise CloudinaryException, "url not given and no retrieve callback given" if options[:url].nil? && self.retrieve.nil?
     raise CloudinaryException, "id not given and retrieve or complete callback given" if options[:id].nil? && (!self.retrieve.nil? || !self.complete.nil?)
 
     debug("Process: #{options.inspect}")
     start
-    process_results    
+    process_results
     wait_for_queue
     options = options.dup
     id = options.delete(:id)
     url = options.delete(:url)
     public_id = options.delete(:public_id)
     row = {
-      "internal_id"=>id, 
-      "url"=>url, 
-      "public_id"=>public_id, 
+      "internal_id"=>id,
+      "url"=>url,
+      "public_id"=>public_id,
       "metadata"=>options.to_json,
-      "status"=>"processing"      
-    }    
+      "status"=>"processing"
+    }
     begin
       insert_row(row)
       add_to_work_queue(row)
@@ -112,7 +111,7 @@ class Cloudinary::Migrator
       raise if !@ignore_duplicates
     end
   end
-      
+
   def done
     start
     process_all_pending
@@ -122,37 +121,37 @@ class Cloudinary::Migrator
     @db.close
     File.delete(@dbfile) if @delete_after_done
   end
-  
+
   def max_given_id
     db.get_first_value("select max(internal_id) from queue").to_i
   end
-  
+
   def close_if_needed(file)
     if file.nil?
       # Do nothing.
-    elsif file.respond_to?(:close!) 
-      file.close! 
+    elsif file.respond_to?(:close!)
+      file.close!
     elsif file.respond_to?(:close)
       file.close
     end
   rescue
     # Ignore errors in closing files
-  end  
+  end
 
-  def temporary_file(data, filename)  
-    file = RUBY_VERSION == "1.8.7" ? Tempfile.new('cloudinary') : Tempfile.new('cloudinary', :encoding => 'ascii-8bit')
+  def temporary_file(data, filename)
+    file = Tempfile.new('cloudinary', :encoding => 'ascii-8bit')
     file.unlink
     file.write(data)
     file.rewind
-    # Tempfile return path == nil after unlink, which break rest-client              
+    # Tempfile return path == nil after unlink, which break rest-client
     class << file
       attr_accessor :original_filename
       def content_type
-        "application/octet-stream"                  
+        "application/octet-stream"
       end
     end
     file.original_filename = filename
-    file                  
+    file
   end
 
   private
@@ -160,21 +159,21 @@ class Cloudinary::Migrator
   def update_all(values)
     @db.execute("update queue set #{values.keys.map{|key| "#{key}=?"}.join(",")}", *values.values)
   end
-  
-  def update_row(row, values)    
+
+  def update_row(row, values)
     values.merge!("updated_at"=>Time.now.to_i)
     query = ["update queue set #{values.keys.map{|key| "#{key}=?"}.join(",")} where id=?"] + values.values + [row["id"]]
     @db.execute(*query)
     values.each{|key, value| row[key.to_s] = value}
-    row    
+    row
   end
-  
+
   def insert_row(values)
     values.merge!("updated_at"=>Time.now.to_i)
     @db.execute("insert into queue (#{values.keys.join(",")}) values (#{values.keys.map{"?"}.join(",")})", *values.values)
     values["id"] = @db.last_insert_row_id
   end
-  
+
   def refill_queue(last_id)
     @db.execute("select * from queue where status in ('error', 'processing') and id > ? limit ?", last_id, 10000) do
       |row|
@@ -183,25 +182,25 @@ class Cloudinary::Migrator
       add_to_work_queue(row)
     end
     last_id
-  end 
+  end
 
   def process_results
     while self.results.length > 0
       row = self.results.pop
-      result = json_decode(row["result"])      
+      result = json_decode(row["result"])
       debug("Done ID=#{row['internal_id']}, result=#{result.inspect}")
-      complete.call(row["internal_id"], result) if complete 
-      if result["error"]        
+      complete.call(row["internal_id"], result) if complete
+      if result["error"]
         status = case result["error"]["http_code"]
         when 400, 404 then "fatal" # Problematic request. Not a server problem.
         else "error"
         end
       else
         status = "completed"
-      end       
+      end
       updates = {:status=>status, :result=>row["result"]}
       updates["public_id"] = result["public_id"] if result["public_id"] && !row["public_id"]
-      begin 
+      begin
         update_row(row, updates)
       rescue SQLite3::ConstraintException
         updates = {:status=>"error", :result=>{:error=>{:message=>"public_id already exists"}}.to_json}
@@ -219,16 +218,16 @@ class Cloudinary::Migrator
       raise if retry_count > tries
       sleep rand * 3
       retry
-    end  
+    end
   end
-  
+
   def start
     return if @started
     @started = true
     @terminate = false
-    
+
     self.work.clear
-    
+
     main = self
     Thread.abort_on_exception = true
     1.upto(@threads) do
@@ -251,8 +250,8 @@ class Cloudinary::Migrator
               elsif defined?(::CarrierWave) && defined?(Cloudinary::CarrierWave) && data.is_a?(Cloudinary::CarrierWave)
                 cw = true
                 begin
-                  data.model.save! 
-                rescue Cloudinary::CarrierWave::UploadError 
+                  data.model.save!
+                rescue Cloudinary::CarrierWave::UploadError
                   # upload errors will be handled by the result values.
                 end
                 result = data.metadata
@@ -264,22 +263,22 @@ class Cloudinary::Migrator
               elsif data.match(/^https?:/)
                 url = data
               else
-                file = main.temporary_file(data, row["public_id"] || "cloudinaryfile") 
+                file = main.temporary_file(data, row["public_id"] || "cloudinaryfile")
               end
             end
-            
+
             if url || file
               options = main.extra_options.merge(:public_id=>row["public_id"])
               json_decode(row["metadata"]).each do
                 |key, value|
                 options[key.to_sym] = value
               end
-                          
+
               result = Cloudinary::Uploader.upload(url || file, options.merge(:return_error=>true)) || ({:error=>{:message=>"Received nil from uploader!"}})
             elsif cw
               result ||= {"status" => "saved"}
             else
-              result = {"error" => {"message" => "Empty data and url", "http_code"=>404}} 
+              result = {"error" => {"message" => "Empty data and url", "http_code"=>404}}
             end
             main.results << {"id"=>row["id"], "internal_id"=>row["internal_id"], "result"=>result.to_json}
           rescue => e
@@ -289,14 +288,14 @@ class Cloudinary::Migrator
           ensure
             main.mutex.synchronize{main.in_process -= 1}
             main.close_if_needed(file)
-          end          
+          end
         end
       end
-    end 
-    
+    end
+
     retry_previous_queue # Retry all work from previous iteration before we start processing this one.
   end
-  
+
   def debug(message)
     if @debug
       mutex.synchronize{
@@ -310,60 +309,60 @@ class Cloudinary::Migrator
     begin
       prev_last_id, last_id = last_id, refill_queue(last_id)
     end while last_id > prev_last_id
-    process_results    
+    process_results
   end
-  
+
   def process_all_pending
     # Waiting for work to finish. While we are at it, process results.
-    while self.in_process > 0      
+    while self.in_process > 0
       process_results
       sleep 0.1
     end
     # Make sure we processed all the results
     process_results
   end
-  
+
   def add_to_work_queue(row)
     self.work << row
     mutex.synchronize{self.in_process += 1}
   end
-  
+
   def wait_for_queue
     # Waiting f
     while self.work.length > @max_processing
-      process_results    
+      process_results
       sleep 0.1
-    end    
+    end
   end
 
-  def self.sample        
+  def self.sample
     migrator = Cloudinary::Migrator.new(
-      :retrieve=>proc{|id| Post.find(id).data}, 
+      :retrieve=>proc{|id| Post.find(id).data},
       :complete=>proc{|id, result| a}
       )
-    
+
     Post.find_each(:conditions=>["id > ?", migrator.max_given_id], :select=>"id") do
       |post|
       migrator.process(:id=>post.id, :public_id=>"post_#{post.id}")
     end
     migrator.done
-  end 
-    
-  def self.test  
+  end
+
+  def self.test
     posts = {}
-    done = {}      
+    done = {}
     migrator = Cloudinary::Migrator.new(
-      :retrieve=>proc{|id| posts[id]}, 
+      :retrieve=>proc{|id| posts[id]},
       :complete=>proc{|id, result| $stderr.print "done #{id} #{result}\n"; done[id] = result}
       )
     start = migrator.max_given_id + 1
     (start..1000).each{|i| posts[i] = "hello#{i}"}
-    
+
     posts.each do
       |id, data|
       migrator.process(:id=>id, :public_id=>"post_#{id}")
     end
     migrator.done
     pp [done.length, start]
-  end        
+  end
 end

data/lib/cloudinary/uploader.rb

@@ -1,83 +1,53 @@
 # Copyright Cloudinary
-require 'rest_client'
+require 'faraday'
 require 'json'
 require 'cloudinary/cache'
 
 class Cloudinary::Uploader
-
-  REMOTE_URL_REGEX = Cloudinary::Utils::REMOTE_URL_REGEX
-  # @deprecated use {Cloudinary::Utils.build_eager} instead
-  def self.build_eager(eager)
-    Cloudinary::Utils.build_eager(eager)
-  end
-
-  # @private
-  def self.build_upload_params(options)
+  @adapter = nil
+  def self.build_upload_params(options, as_bool = false)
     #symbolize keys
     options = options.clone
     options.keys.each { |key| options[key.to_sym] = options.delete(key) if key.is_a?(String) }
 
     params = {
-      :access_control                       => Cloudinary::Utils.json_array_param(options[:access_control]),
-      :access_mode                          => options[:access_mode],
-      :allowed_formats                      => Cloudinary::Utils.build_array(options[:allowed_formats]).join(","),
-      :asset_folder                         => options[:asset_folder],
-      :async                                => Cloudinary::Utils.as_safe_bool(options[:async]),
-      :auto_tagging                         => options[:auto_tagging] && options[:auto_tagging].to_f,
-      :background_removal                   => options[:background_removal],
-      :backup                               => Cloudinary::Utils.as_safe_bool(options[:backup]),
-      :callback                             => options[:callback],
-      :categorization                       => options[:categorization],
-      :cinemagraph_analysis                 => Cloudinary::Utils.as_safe_bool(options[:cinemagraph_analysis]),
-      :colors                               => Cloudinary::Utils.as_safe_bool(options[:colors]),
-      :context                              => Cloudinary::Utils.encode_context(options[:context]),
-      :custom_coordinates                   => Cloudinary::Utils.encode_double_array(options[:custom_coordinates]),
-      :detection                            => options[:detection],
-      :discard_original_filename            => Cloudinary::Utils.as_safe_bool(options[:discard_original_filename]),
-      :display_name                         => options[:display_name],
-      :eager                                => Cloudinary::Utils.build_eager(options[:eager]),
-      :eager_async                          => Cloudinary::Utils.as_safe_bool(options[:eager_async]),
-      :eager_notification_url               => options[:eager_notification_url],
-      :exif                                 => Cloudinary::Utils.as_safe_bool(options[:exif]),
-      :eval                                 => options[:eval],
-      :on_success                           => options[:on_success],
-      :face_coordinates                     => Cloudinary::Utils.encode_double_array(options[:face_coordinates]),
-      :faces                                => Cloudinary::Utils.as_safe_bool(options[:faces]),
-      :folder                               => options[:folder],
-      :format                               => options[:format],
-      :filename_override                    => options[:filename_override],
-      :headers                              => build_custom_headers(options[:headers]),
-      :image_metadata                       => Cloudinary::Utils.as_safe_bool(options[:image_metadata]),
-      :media_metadata                       => Cloudinary::Utils.as_safe_bool(options[:media_metadata]),
-      :invalidate                           => Cloudinary::Utils.as_safe_bool(options[:invalidate]),
-      :moderation                           => options[:moderation],
-      :notification_url                     => options[:notification_url],
-      :ocr                                  => options[:ocr],
-      :overwrite                            => Cloudinary::Utils.as_safe_bool(options[:overwrite]),
-      :phash                                => Cloudinary::Utils.as_safe_bool(options[:phash]),
-      :proxy                                => options[:proxy],
-      :public_id                            => options[:public_id],
-      :public_id_prefix                     => options[:public_id_prefix],
-      :quality_analysis                     => Cloudinary::Utils.as_safe_bool(options[:quality_analysis]),
-      :quality_override                     => options[:quality_override],
-      :raw_convert                          => options[:raw_convert],
-      :responsive_breakpoints               => Cloudinary::Utils.generate_responsive_breakpoints_string(options[:responsive_breakpoints]),
-      :return_delete_token                  => Cloudinary::Utils.as_safe_bool(options[:return_delete_token]),
-      :similarity_search                    => options[:similarity_search],
-      :visual_search                        => Cloudinary::Utils.as_safe_bool(options[:visual_search]),
-      :tags                                 => options[:tags] && Cloudinary::Utils.build_array(options[:tags]).join(","),
-      :timestamp                            => (options[:timestamp] || Time.now.to_i),
-      :transformation                       => Cloudinary::Utils.generate_transformation_string(options.clone),
-      :type                                 => options[:type],
-      :unique_filename                      => Cloudinary::Utils.as_safe_bool(options[:unique_filename]),
-      :upload_preset                        => options[:upload_preset],
-      :use_filename                         => Cloudinary::Utils.as_safe_bool(options[:use_filename]),
-      :use_filename_as_display_name         => Cloudinary::Utils.as_safe_bool(options[:use_filename_as_display_name]),
-      :use_asset_folder_as_public_id_prefix => Cloudinary::Utils.as_safe_bool(options[:use_asset_folder_as_public_id_prefix]),
-      :unique_display_name                  => Cloudinary::Utils.as_safe_bool(options[:unique_display_name]),
-      :accessibility_analysis               => Cloudinary::Utils.as_safe_bool(options[:accessibility_analysis]),
-      :metadata                             => Cloudinary::Utils.encode_context(options[:metadata])
+      :access_control         => Cloudinary::Utils.json_array_param(options[:access_control]),
+      :allowed_formats        => Cloudinary::Utils.build_array(options[:allowed_formats]).join(","),
+      :auto_tagging           => options[:auto_tagging] && options[:auto_tagging].to_f,
+      :context                => Cloudinary::Utils.encode_context(options[:context]),
+      :custom_coordinates     => Cloudinary::Utils.encode_double_array(options[:custom_coordinates]),
+      :eager                  => Cloudinary::Utils.build_eager(options[:eager]),
+      :face_coordinates       => Cloudinary::Utils.encode_double_array(options[:face_coordinates]),
+      :headers                => build_custom_headers(options[:headers]),
+      :responsive_breakpoints => Cloudinary::Utils.generate_responsive_breakpoints_string(options[:responsive_breakpoints]),
+      :tags                   => options[:tags] && Cloudinary::Utils.build_array(options[:tags]).join(","),
+      :timestamp              => (options[:timestamp] || Time.now.to_i),
+      :transformation         => Cloudinary::Utils.generate_transformation_string(options.clone),
+      :metadata               => Cloudinary::Utils.encode_context(options[:metadata])
     }
+
+    bool_params = [
+      :async, :backup, :cinemagraph_analysis, :colors, :discard_original_filename, :eager_async, :exif, :faces,
+      :image_metadata, :media_metadata, :invalidate, :overwrite, :phash, :quality_analysis, :return_delete_token,
+      :visual_search, :unique_filename, :use_filename, :use_filename_as_display_name,
+      :use_asset_folder_as_public_id_prefix, :unique_display_name, :accessibility_analysis
+    ]
+
+    string_params = [
+      :access_mode, :asset_folder, :background_removal, :callback, :categorization, :detection, :display_name,
+      :eager_notification_url, :eval, :on_success, :folder, :format, :filename_override, :moderation, :notification_url,
+      :ocr, :proxy, :public_id, :public_id_prefix, :quality_override, :raw_convert, :similarity_search, :type,
+      :upload_preset
+    ]
+
+    bool_params.each do |b|
+      params[b] =  as_bool ? Cloudinary::Utils.as_bool(options[b]): Cloudinary::Utils.as_safe_bool(options[b])
+    end
+
+    string_params.each do |s|
+      params[s] =  options[s]
+    end
+
     params
   end
 
@@ -394,29 +364,39 @@ class Cloudinary::Uploader
       params[:signature]  = Cloudinary::Utils.api_sign_request(params.reject { |k, v| non_signable.include?(k) }, api_secret, signature_algorithm)
       params[:api_key]    = api_key
     end
-    proxy   = options[:api_proxy] || Cloudinary.config.api_proxy
-    timeout = options.fetch(:timeout) { Cloudinary.config.to_h.fetch(:timeout, 60) }
-
-    result = nil
-
-    api_url = Cloudinary::Utils.cloudinary_api_url(action, options)
-    RestClient::Request.execute(:method => :post, :url => api_url, :payload => params.reject { |k, v| v.nil? || v=="" }, :timeout => timeout, :headers => headers, :proxy => proxy) do
-    |response, request, tmpresult|
-      raise CloudinaryException, "Server returned unexpected status code - #{response.code} - #{response.body}" unless [200, 400, 401, 403, 404, 500].include?(response.code)
-      begin
-        result = Cloudinary::Utils.json_decode(response.body)
-      rescue => e
-        # Error is parsing json
-        raise CloudinaryException, "Error parsing server response (#{response.code}) - #{response.body}. Got - #{e}"
-      end
-      if result["error"]
-        if return_error
-          result["error"]["http_code"] = response.code
-        else
-          raise CloudinaryException, result["error"]["message"]
-        end
+
+    api_url   = Cloudinary::Utils.cloudinary_api_url(action, options)
+    api_proxy = options[:api_proxy] || Cloudinary.config.api_proxy
+    timeout   = options.fetch(:timeout) { Cloudinary.config.to_h.fetch(:timeout, 60) }
+
+    conn =  Faraday.new(url: api_url) do |faraday|
+      faraday.proxy = api_proxy if api_proxy
+      faraday.request :multipart, **options
+      faraday.request :url_encoded
+      faraday.adapter @adapter || Faraday.default_adapter
+    end
+
+    response = conn.send(:post) do |req|
+      req.headers = headers
+      req.body = params.reject { |_, v| v.nil? || v=="" }
+      req.options.timeout = timeout if timeout
+    end
+
+    raise CloudinaryException, "Server returned unexpected status code - #{response.status} - #{response.body}" unless [200, 400, 401, 403, 404, 500].include?(response.status)
+    begin
+      result = Cloudinary::Utils.json_decode(response.body)
+    rescue => e
+      # Error is parsing json
+      raise CloudinaryException, "Error parsing server response (#{response.status}) - #{response.body}. Got - #{e}"
+    end
+    if result["error"]
+      if return_error
+        result["error"]["http_code"] = response.status
+      else
+        raise CloudinaryException, result["error"]["message"]
       end
     end
+
     if use_cache && !result.nil?
       cache_results(result)
     end