cnvrg 1.9.9.9 → 1.9.9.9.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9c3589f491e89ed7e76b625e215b48cd98af828e2de0c3015435d452717740c3
-  data.tar.gz: a32b3b3dac6a0932a5dffe4a900a4f026b01ae524c5f60bc716d44553565eaa4
+  metadata.gz: abe86787c977039f70850a0c0a5dbe203930f5cf5a6bd20dc6c3b8f8ed6cb21c
+  data.tar.gz: 4fb12784ae1d4452d71dbd069cc3be2898b3244401a801339b72ce0102da24e5
 SHA512:
-  metadata.gz: a9aa02bb0bcb739fe99f423d97bbec6536405588979f6faa5cda6054803419c5112b22a358f833e96981caa912e9e30d65db3cb4554b4d734dd27f59a65d8c45
-  data.tar.gz: 56bf19f2bdde12f8bb3a3f17efe185ecd5ca7edd7050e3fefd0bcb7fa237f800dbbbafb78974135e3e576396aa9d1685e44be19c3ca8491ad8ce90fa6c5fa545
+  metadata.gz: 2663d0d5c0733b7fff8d20fe798fe395f5df856b444af9f51253bc31b0fefd380d78e616651ee0b4280b6d29dc9bac7564d6cd4b7c7835ad190e1e5966215b7e
+  data.tar.gz: afe2c81bb5f884692dc5cd16b5984ab30a60c4ad27c434ed16462e307fcf867ecfaedc9e78034fb3427a1d1fc7e7bca1b47237e803fe2ec36601dac1d313b236

data/lib/cnvrg/cli.rb

@@ -2274,13 +2274,16 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true)
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=100)
       begin
         # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
         log_start(__method__, args, options)
         @project = Project.new(get_project_home)
+        chunk_size = chunk_size ? chunk_size : options["chunk_size"]
+
 
         # Enable local/experiment exception logging
         suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
@@ -2396,7 +2399,7 @@ module Cnvrg
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
 
-        buffered_errors = @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions)
+        buffered_errors = @files.upload_mrultiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions, chunk_size: chunk_size)
         @files.delete_files_from_server(deleted, commit_sha1, suppress_exceptions: suppress_exceptions)
 
         progressbar.finish
@@ -2970,7 +2973,7 @@ module Cnvrg
     method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
-    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -2995,7 +2998,8 @@ module Cnvrg
       end
       invoke :upload, [false, true,  direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
-             :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"], :debug_mode => options['debug_mode'], :git_diff => options["git_diff"]
+             :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
+             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"]
 
     end
 
@@ -3288,7 +3292,7 @@ module Cnvrg
                       puts line
                     end
                     log << cur_log
-                    if log.size >= 1
+                    if log.size >= 10
                       @exp.upload_temp_log(log) unless log.empty?
                       log = []
                       elsif (start_time + 15.seconds) <= Time.now

data/lib/cnvrg/files.rb

@@ -86,7 +86,7 @@ module Cnvrg
       end
     end
 
-    def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false)
+    def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false, chunk_size: 100)
       #open files on the server.
       Cnvrg::Logger.log_info("Uploading project files")
       return if files_list.blank?
@@ -95,56 +95,62 @@ module Cnvrg
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
 
-      files_list = files_list.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
-      resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
-      unless Cnvrg::CLI.is_response_success(resp, false)
-        raise SignalException.new("Cant upload files to the server.")
-      end
-      # resolve bucket
-      res = resp['result']
-      files = res['files']
-
-      #upload files
-      token_mutex = Mutex.new
+      blob_ids = []
       buffered_errors = {}
-      blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
-
-        token_mutex.synchronize {
-          refresh_storage_token
-        }
-
-        begin
-          Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
-        rescue => e
-          Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
-          Cnvrg::Logger.log_error(e)
-          Cnvrg::Logger.log_method(bind: binding)
-
-          buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
-
-          raise e unless suppress_exceptions
+      files_list.each_slice(chunk_size).each do |chunk_of_files|
+        Cnvrg::Logger.log_info("Upload chunk")
+        parsed_chunk_of_files = chunk_of_files.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
+
+        resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {
+            files: parsed_chunk_of_files,
+            commit: commit_sha1
+        })
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          raise SignalException.new("Cant upload files to the server.")
         end
-        progress.progress += 1 if progress.present?
+        # resolve bucket
+        res = resp['result']
+        files = res['files']
+
+        #upload files
+        blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+          begin
+            Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
+          rescue => e
+            Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
+            Cnvrg::Logger.log_error(e)
+            Cnvrg::Logger.log_method(bind: binding)
+
+            buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
+
+            raise e unless suppress_exceptions
+          end
+          progress.progress += 1 if progress.present?
 
-        unless buffered_errors.key?(file)
-          files[file]["bv_id"]
-        else
-          nil
+          unless buffered_errors.key?(file)
+            files[file]["bv_id"]
+          else
+            nil
+          end
         end
-      end
 
-      # remove nil files (failed files) from blob_ids
-      blob_ids.compact!
 
-      #save files on the server.
-      resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
-      unless Cnvrg::CLI.is_response_success(resp, false)
-        raise SignalException.new("Cant save uploaded files to the server.")
-      end
+      blob_ids.concat blob_id_chunk
+    end
+
+    # remove nil files (failed files) from blob_ids
+    blob_ids.compact!
 
-      return buffered_errors
+    #save files on the server.
+    resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
+    unless Cnvrg::CLI.is_response_success(resp, false)
+      raise SignalException.new("Cant save uploaded files to the server.")
     end
 
+    return buffered_errors
+  end
+
+
 
     def delete_files_from_server_old(files, commit_sha1)
       #files are absolute path here.

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.9.9.9'
+  VERSION = '1.9.9.9.4'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.9.9.9
+  version: 1.9.9.9.4
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler