cnvrg 1.9.9.6 → 1.9.9.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd47735837c5b994e5a7b614ac38dacabb094872e4edcefe92374a726874f6e4
-  data.tar.gz: 63029e65065352cf285d1321e73e2f51bab63518460a6ffcd69920c80a4ea833
+  metadata.gz: c2b4e4f9fec5ba43fc66cec1bafd254e9c69911dc8b5f7a786f2628a7460b12a
+  data.tar.gz: 262df246fbaae19a9c20fddfddb7e39f1a9813e50588951d31bcde8988d456e4
 SHA512:
-  metadata.gz: 8c202546702f4c041f1d2bad1443240f35168b0a5c489503f1d9852e612f49ffd60a6b68c8066a02e39ba8e131451166121afeb68a063a5f040d1d9938a805ab
-  data.tar.gz: 6e5296dda882e2d9dcf9d06ec72b6c0ffe69a089aa82f846846dd7daa46f9ff698e25da3ce7fb0eb96d3c9488803d022d06065bcc7791adfa8f8bc75a1ce1cf5
+  metadata.gz: 554850cc31a667e7a5494833c6cbedbd38e40e0cc9ebd4a0d13904c6edb33f25b0655d76f837eb2a0912bf4a509f3dea6ef2591897f272ee9e4e0f39f82f04f0
+  data.tar.gz: 5ded18aca1079b010efbd7f8fb025debe9fc6af17621286db1142932a665069effea73d75f7bd8188a02e4e8f4ef59c19ba8baf525877a0282fae1051917db4d

data/lib/cnvrg/cli.rb

@@ -2274,13 +2274,16 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true)
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=100)
       begin
         # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
         log_start(__method__, args, options)
         @project = Project.new(get_project_home)
+        chunk_size = chunk_size ? chunk_size : options["chunk_size"]
+
 
         # Enable local/experiment exception logging
         suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
@@ -2396,7 +2399,7 @@ module Cnvrg
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
 
-        buffered_errors = @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions)
+        buffered_errors = @files.upload_mrultiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions, chunk_size: chunk_size)
         @files.delete_files_from_server(deleted, commit_sha1, suppress_exceptions: suppress_exceptions)
 
         progressbar.finish
@@ -2970,7 +2973,7 @@ module Cnvrg
     method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
     method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
-    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
 
     def sync(direct = true)
       verify_logged_in(true) if direct
@@ -2995,7 +2998,8 @@ module Cnvrg
       end
       invoke :upload, [false, true,  direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
-             :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"], :debug_mode => options['debug_mode'], :git_diff => options["git_diff"]
+             :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
+             :debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"]
 
     end
 
@@ -3288,7 +3292,7 @@ module Cnvrg
                       puts line
                     end
                     log << cur_log
-                    if log.size >= 1
+                    if log.size >= 10
                       @exp.upload_temp_log(log) unless log.empty?
                       log = []
                       elsif (start_time + 15.seconds) <= Time.now

data/lib/cnvrg/datafiles.rb

@@ -168,7 +168,7 @@ module Cnvrg
         error = nil
         upload_resp = nil
         10.times do
-          upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, override: force, is_branch: new_branch, partial_commit: partial_commit})
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, override: force, is_branch: new_branch, partial_commit: partial_commit})
           if Cnvrg::CLI.is_response_success(upload_resp, false)
             error = nil
             break
@@ -308,6 +308,7 @@ module Cnvrg
             commit_sha1: commit_sha1,
             tree: tree,
             override: override,
+            force: override,
             is_branch: new_branch,
             partial_commit: partial_commit
         })

data/lib/cnvrg/files.rb

@@ -86,7 +86,7 @@ module Cnvrg
       end
     end
 
-    def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false)
+    def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false, chunk_size: 100)
       #open files on the server.
       Cnvrg::Logger.log_info("Uploading project files")
       return if files_list.blank?
@@ -95,56 +95,62 @@ module Cnvrg
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
 
-      files_list = files_list.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
-      resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
-      unless Cnvrg::CLI.is_response_success(resp, false)
-        raise SignalException.new("Cant upload files to the server.")
-      end
-      # resolve bucket
-      res = resp['result']
-      files = res['files']
-
-      #upload files
-      token_mutex = Mutex.new
+      blob_ids = []
       buffered_errors = {}
-      blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
-
-        token_mutex.synchronize {
-          refresh_storage_token
-        }
-
-        begin
-          Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
-        rescue => e
-          Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
-          Cnvrg::Logger.log_error(e)
-          Cnvrg::Logger.log_method(bind: binding)
-
-          buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
-
-          raise e unless suppress_exceptions
+      files_list.each_slice(chunk_size).each do |chunk_of_files|
+        Cnvrg::Logger.log_info("Upload chunk")
+        parsed_chunk_of_files = chunk_of_files.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
+
+        resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {
+            files: parsed_chunk_of_files,
+            commit: commit_sha1
+        })
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          raise SignalException.new("Cant upload files to the server.")
         end
-        progress.progress += 1 if progress.present?
+        # resolve bucket
+        res = resp['result']
+        files = res['files']
+
+        #upload files
+        blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+          begin
+            Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
+          rescue => e
+            Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
+            Cnvrg::Logger.log_error(e)
+            Cnvrg::Logger.log_method(bind: binding)
+
+            buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
+
+            raise e unless suppress_exceptions
+          end
+          progress.progress += 1 if progress.present?
 
-        unless buffered_errors.key?(file)
-          files[file]["bv_id"]
-        else
-          nil
+          unless buffered_errors.key?(file)
+            files[file]["bv_id"]
+          else
+            nil
+          end
         end
-      end
 
-      # remove nil files (failed files) from blob_ids
-      blob_ids.compact!
 
-      #save files on the server.
-      resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
-      unless Cnvrg::CLI.is_response_success(resp, false)
-        raise SignalException.new("Cant save uploaded files to the server.")
-      end
+      blob_ids.concat blob_id_chunk
+    end
+
+    # remove nil files (failed files) from blob_ids
+    blob_ids.compact!
 
-      return buffered_errors
+    #save files on the server.
+    resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
+    unless Cnvrg::CLI.is_response_success(resp, false)
+      raise SignalException.new("Cant save uploaded files to the server.")
     end
 
+    return buffered_errors
+  end
+
+
 
     def delete_files_from_server_old(files, commit_sha1)
       #files are absolute path here.

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.9.9.6'
+  VERSION = '1.9.9.9.5'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.9.9.6
+  version: 1.9.9.9.5
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-20 00:00:00.000000000 Z
+date: 2020-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.