cnvrg 1.6.21 → 1.6.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 82961f200868479b73c2b80dfb2fc4a0973b2ed52798ae6fdd2d3296d3f815c0
-  data.tar.gz: 1ecf89c6eaa659c2f8479353bf4a2c564655e5d6c50ad922551889672bedf70f
+  metadata.gz: edee87a5b2d2c1693d36ddcfeb992b4e87a88521988e6335844ab522401cee46
+  data.tar.gz: bd0286b12dc83235c57bcf64726d2af6860608e0a061c885888bf044129d1ab0
 SHA512:
-  metadata.gz: 23cb859d1ccdfd397c3942655b1e41fc651f5596d92ca0c90c552b03b0d178fd207272139bbcba2381dc4e69480a6ada5e072ac607f8c20b78e71d8725d34d8c
-  data.tar.gz: 51f782aa51de9b3c46af30c320bfe6344eb0eb86104461cf675cfa5ecafb870b580147a51dd00a35de528f603b2fd60af116387a5dc56eddb5122bd76215c058
+  metadata.gz: e1421a63d3a34350f181deaead9c9b5f1cebe91f75f675a81d2a0cb49de0990d690efc08ba9e8ef74315b44041889ed093e19fbf4a9a271ec0f1394b39c45748
+  data.tar.gz: 67872a7abe1dbac6d001c28c4a556c0b8c09b153dade45eb6a6e54ed318a804235e089504c5e2e409cef27dde5235ea1ab35d7a239bc7aefad976ce60969e111

data/lib/cnvrg/cli.rb

@@ -1212,16 +1212,10 @@ module Cnvrg
         dir = dir[0..-2] if dir.end_with? '/'
         dir = dir[1..-1] if dir.start_with? '/'
 
-        progressbar = ProgressBar.create(:title => "Upload Progress",
-                                         :progress_mark => '=',
-                                         :format => "%b>>%i| %p%% %t",
-                                         :starting_at => 0,
-                                         :total => @files.size,
-                                         :autofinish => true)
         @files.each_slice(chunk_size).each do |list_files|
           temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
           #will throw a signal exception if something goes wrong.
-          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, progressbar: progressbar, prefix: dir)
+          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, prefix: dir, total: @files.size)
         end
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
@@ -1951,12 +1945,11 @@ module Cnvrg
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
       if !force and !init
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
-        invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
+        total_deleted, total_downloaded = invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
-
       # w(new_branch,  verbose,sync,force, tags, chunk_size)
-      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message], :new_branch=>new_branch,
-             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
+      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message, total_deleted: total_deleted, total_downloaded: total_downloaded],
+             :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
 
     end
     desc 'upload_data_new', 'upload_data_new', :hide => true
@@ -1968,31 +1961,69 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
-    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil)
+    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
       begin
-      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
-      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
-      if upload_res < 0
-        return
-      end
-      invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res], :new_branch=>new_branch, :force =>force
-      if tags
-        log_message('Uploading Tags', Thor::Shell::Color::BLUE)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-        begin
-          tag_file = File.open(options[:tags], "r+")
-          status = @dataset.upload_tags_via_yml(tag_file)
-        rescue
-          log_message('Tags file not found', Thor::Shell::Color::RED)
+        commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
+        files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
+
+        upload_size = files_to_upload + upload_errors.try(:size) rescue 0
+
+        if files_to_upload <= 0
+          check = Helpers.checkmark
+          log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
+          if total_deleted > 0
+            log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
+          end
+
+          if total_downloaded > 0
+            log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
+          end
+
+          if upload_errors.try(:size) > 0
+            log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
+            upload_errors.each do |file_hash|
+              log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
+            end
+          end
+
+          if upload_errors.blank? and total_downloaded == 0 and total_deleted == 0
+            Cnvrg::CLI.log_message("You are up to date", Thor::Shell::Color::GREEN)
+          end
           return
         end
-        if status
-          log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
-        else
-          log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
+        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload], :new_branch=>new_branch, :force =>force
+        if tags
+          log_message('Uploading Tags', Thor::Shell::Color::BLUE)
+          dataset_dir = is_cnvrg_dir(Dir.pwd)
+          @dataset = Dataset.new(dataset_dir)
+          begin
+            tag_file = File.open(options[:tags], "r+")
+            status = @dataset.upload_tags_via_yml(tag_file)
+          rescue
+            log_message('Tags file not found', Thor::Shell::Color::RED)
+            return
+          end
+          if status
+            log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
+          else
+            log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
+          end
+        end
+        if total_deleted > 0
+          log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
+        end
+
+        if total_downloaded > 0
+          log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
+        end
+
+        log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
+        if upload_errors.try(:size) > 0
+          log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
+          upload_errors.each do |file_hash|
+            log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
+          end
         end
-      end
       rescue => e
         Cnvrg::CLI.log_message(e.message, 'red')
         Cnvrg::Logger.log_error(e)
@@ -2022,17 +2053,16 @@ module Cnvrg
       verify_logged_in(true)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
-      direct = options[:direct]
       new_branch = options["new_branch"] || false
       force = options["force"] || false
       chunk_size = options["chunk_size"] || false
       message = options["message"]
-      commit_sha1 = nil
       @dataset = Dataset.new(dataset_dir)
       @dataset.backup_idx
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       next_commit = @dataset.get_next_commit #if there was a partial commit..
-      chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
+      files_list = @dataset.list_all_files
+      chunks = (files_list.length.to_f / chunk_size).ceil
       resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset, message: message)
       if !resp['result']['can_commit']
         log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
@@ -2043,7 +2073,7 @@ module Cnvrg
         @dataset.set_partial_commit(next_commit)
       end
       @dataset.set_next_commit(commit_sha1)
-      return commit_sha1
+      return commit_sha1, files_list
     end
 
 
@@ -2066,7 +2096,7 @@ module Cnvrg
             @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+            log_message("#{check} Data sync finished", Thor::Shell::Color::GREEN)
             @dataset.remove_next_commit #takes the next commit and put it as current commit
             @dataset.set_partial_commit(nil)
             @dataset.backup_idx
@@ -2119,55 +2149,53 @@ module Cnvrg
     # method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
-    def upload_data_files(new_commit, *files)
+    def upload_data_files(new_commit, files_list: [])
       begin
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
-      new_commit ||= @dataset.get_next_commit
-      partial_commit = @dataset.get_partial_commit
-      if new_commit.blank?
-        log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
-        return false
-      end
-      force = options[:force] || false
-      chunk_size = options[:chunk_size]
-      chunk_size = [chunk_size, 1].max
-      new_branch = options["new_branch"] || false
-      new_tree = {}
-      force = options["force"] || false
-      parallel_threads = options["parallel"] || ParallelThreads
-      all_files = @dataset.list_all_files
-      progressbar = ProgressBar.create(:title => "Upload Progress",
-                                       :progress_mark => '=',
-                                       :format => "%b>>%i| %p%% %t",
-                                       :starting_at => 0,
-                                       :total => all_files.length,
-                                       :autofinish => true)
-      files_uploaded = 0
-      all_files.each_slice(chunk_size).each do |list_files|
-        Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
-        temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
-        upload_resp = @files.upload_multiple_files(new_commit, temp_tree,
-                                                   threads: parallel_threads,
-                                                   force: force,
-                                                   new_branch: new_branch,
-                                                   progressbar: progressbar,
-                                                   partial_commit: partial_commit)
-
-        files_uploaded += upload_resp
-        temp_tree.each do |k, v|
-          new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
-        end
-      end
-      @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
+        new_commit ||= @dataset.get_next_commit
+        partial_commit = @dataset.get_partial_commit
+        if new_commit.blank?
+          log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
+          return false
+        end
+        chunk_size = options[:chunk_size]
+        chunk_size = [chunk_size, 1].max
+        new_branch = options["new_branch"] || false
+        new_tree = {}
+        force = options["force"] || false
+        parallel_threads = options["parallel"] || ParallelThreads
+        all_files = files_list
+        all_files = @dataset.list_all_files if files_list.blank?
+        files_uploaded = 0
+        upload_errors = []
+
+        all_files.each_slice(chunk_size).each do |list_files|
+          Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
+          temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+          upload_resp, upload_error_files = @files.upload_multiple_files(new_commit, temp_tree,
+                                                                         threads: parallel_threads,
+                                                                         force: force,
+                                                                         new_branch: new_branch,
+                                                                         partial_commit: partial_commit,
+                                                                         total: all_files.length)
+
+          files_uploaded += upload_resp
+          upload_errors += upload_error_files if upload_error_files.present?
+          temp_tree.each do |k, v|
+            new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
+          end
+        end
+
+        @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
       rescue => e
         Cnvrg::Logger.log_error(e)
         raise e
       end
-      return files_uploaded
+      return files_uploaded, upload_errors.try(:flatten).try(:compact)
     end
 
 
@@ -2435,52 +2463,51 @@ module Cnvrg
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
-        all_files = all_files
-        @dataset.generate_idx
         res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
         unless CLI.is_response_success(res, false)
           log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
           exit(1)
         end
-        result = res["result"]["tree"]
-        commit = res["result"]["commit"]
-        can_commit = res["result"]["can_commit"] || false #can commit means that our next commit is newer than the latest commit
-        # so if can_commit is true it means that we have to be up-to-date with the dataset.
-        update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
+        result = res["result"]
+        tree = result["tree"]
+        commit = result["commit"]
+        update_total = [tree['added'], tree["updated_on_server"], tree["conflicts"], tree["deleted"]].compact.flatten.size
         successful_changes  = 0
-        if update_total == 0 or can_commit
-          log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
-          return true
-        elsif options["verbose"]
-          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
+        if update_total == 0
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
+          return 0, 0
         else
+          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE, options["verbose"])
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
         end
-        Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
-        Cnvrg::Logger.log_info("Compare idx res: #{result}")
+        Cnvrg::Logger.log_info("Current commit: #{@dataset.last_local_commit}, destination commit: #{commit}")
+        Cnvrg::Logger.log_info("Compare idx res: #{tree}")
         progressbar = ProgressBar.create(:title => "Download Progress",
-                                           :progress_mark => '=',
-                                           :format => "%b>>%i| %p%% %t",
-                                           :starting_at => 0,
-                                           :total => update_total,
-                                           :autofinish => true)
-        conflicts = @files.mark_conflicts(result)
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => update_total,
+                                         :autofinish => true)
 
+        conflicts = @files.mark_conflicts(tree)
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
-        update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
-        added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
-        # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
-        deleted = result["deleted"].to_a
+        update_res = @files.download_files_in_chunks(tree["updated_on_server"], progress: progressbar) if tree["updated_on_server"].present?
+        added_res = @files.download_files_in_chunks(tree["added"], progress: progressbar) if tree["added"].present?
+        deleted = tree["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
-        progressbar.progress += deleted.size if progressbar.present?
+
         if !delete_res
           log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
           exit(1)
         end
+
+        progressbar.progress += deleted.size if progressbar.present? and deleted.size > 0
+
         success = (update_res.blank? or update_res.is_success?)
         success &= (delete_res.blank? or delete_res.is_success?)
         success &= (added_res.blank? or added_res.is_success?)
+
         if success
           # update idx with latest commit
           @dataset.update_idx_with_commit!(commit)
@@ -2490,17 +2517,24 @@ module Cnvrg
             log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
             log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
+            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, !sync)
           end
-          return true
+
+          total_deleted = deleted.try(:size)
+          total_downloaded = tree["added"].try(:size) || 0
+          total_downloaded +=  tree["updated_on_server"].try(:size) if tree["updated_on_server"].present?
+
+          return total_deleted, total_downloaded
+        else
+          return []
         end
       rescue SignalException => e
         Cnvrg::Logger.log_error(e)
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       rescue => e
-        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         Cnvrg::Logger.log_error(e)
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
         exit(1)
       end
     end
@@ -3386,10 +3420,6 @@ module Cnvrg
           output_dir  = "output"
         end
         image = options["image"] || nil
-        if image.blank?
-          image  = "cnvrg"
-        end
-
         forced_commit = nil
         if sync_before and !project.is_git
           if force
@@ -5491,7 +5521,7 @@ module Cnvrg
         end
       end
 
-      def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
+      def self.log_message(message, type = Thor::Shell::Color::BLUE)
         if $LOG.blank?
           ### handle case when $LOG is not initialized
           CLI.new.log_handler
@@ -5513,11 +5543,12 @@ module Cnvrg
           color = nil
           $LOG.info message: message, type: "unknown"
         end
-        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}" if to_print
+        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}"
       end
 
       def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
-        CLI.log_message(message, type, to_print: to_print)
+        return if not to_print
+        CLI.log_message(message, type)
       end
 
       def log_error(e)

data/lib/cnvrg/datafiles.rb

@@ -114,33 +114,52 @@ module Cnvrg
     end
 
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
-        cli=CLI.new
         Cnvrg::Logger.log_info("Sending Upload Files request")
-        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
-        unless Cnvrg::CLI.is_response_success(upload_resp, false)
+        error = nil
+        upload_resp = nil
+        10.times do
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
+          if Cnvrg::CLI.is_response_success(upload_resp, false)
+            error = nil
+            break
+          end
+          error = upload_resp
           Cnvrg::Logger.log_method(bind: binding)
-          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          Cnvrg::CLI.log_info("Got an error message from server, #{upload_resp.try(:fetch, "message")}, trying again")
         end
+        raise Exception.new("Can't upload data files: #{error["message"]}") if error.present?
+
         Cnvrg::Logger.log_info("Uploading files")
         results = upload_resp['result'].with_indifferent_access
+
         if results['files'].blank?
-          progressbar.progress += tree.keys.length if progressbar.present?
-          return 0
+          return 0, []
+        end
+
+        if @temp_upload_progressbar.blank?
+          @temp_upload_progressbar = ProgressBar.create(:title => "Upload Progress",
+                                                        :progress_mark => '=',
+                                                        :format => "%b>>%i| %p%% %t",
+                                                        :starting_at => 0,
+                                                        :total => total,
+                                                        :autofinish => true)
         end
+
         files = results['files']
-        progressbar.progress += tree.keys.length - files.length if progressbar.present?
+        upload_error_files = []
+        @temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
         Parallel.map((files.keys), in_threads: threads) do |k|
           o = tree[k].merge(files[k])
           success = upload_single_file(o)
           if not success
+            upload_error_files << {absolute_path: o[:absolute_path]}
             files.except!(k)
             tree.except!(k)
-            cli.log_message("Error while uploading file: #{o[:absolute_path]}", Thor::Shell::Color::RED)
             Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
           end
-          progressbar.progress += 1 if progressbar.present?
+          @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
         end
         blob_ids = files.values.map {|f| f['bv_id']}
         if blob_ids.present?
@@ -153,25 +172,21 @@ module Cnvrg
           end
         end
         Cnvrg::Logger.log_info("Upload Success")
-        return files.keys.length
+        return files.try(:keys).try(:length), upload_error_files
       rescue => e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
       end
-
     end
 
     def upload_single_file(file)
       begin
-        cli = CLI.new
         file = file.as_json
         Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
-        success = @downloader.upload(file["path"], file["absolute_path"])
-        return false if not success
+        @downloader.safe_upload(file["path"], file["absolute_path"])
         Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
-        cli.log_message(e.message, Thor::Shell::Color::RED)
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
         return false
@@ -815,6 +830,7 @@ module Cnvrg
       end
       return true
     end
+
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -822,9 +838,6 @@ module Cnvrg
       rescue => e
         return Cnvrg::Result.new(false, '')
       end
-
-      return Cnvrg::Result.new(true, '')
-
     end
 
     def download_dir(dataset_home, absolute_path)
@@ -869,6 +882,7 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
+
     def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -876,14 +890,13 @@ module Cnvrg
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
-      Cnvrg::CLI.is_response_success(response, true)
-      return response
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
+                                                                                username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
       rescue => e
         return false
       end
-
     end
 
     def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )

data/lib/cnvrg/dataset.rb

@@ -103,8 +103,12 @@ module Cnvrg
 
     def backup_idx
       Cnvrg::Logger.log_info("Backup idx")
-      idx = self.get_idx
-      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        FileUtils.cp "#{self.local_path}/.cnvrg/idx.yml", "#{self.local_path}/.cnvrg/idx.yml.backup"
+      else
+        idx = {commit: nil, tree: {}}
+        File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
+      end
     end
 
     def restore_idx
@@ -780,14 +784,6 @@ module Cnvrg
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
     end
 
-    def get_current_commit()
-      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
-        return nil
-      end
-      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-      return idx_hash[:commit]
-    end
-
     def compare_commit(commit)
       if commit.nil? or commit.empty?
         commit = last_local_commit

data/lib/cnvrg/downloader/client.rb

@@ -2,6 +2,8 @@
 module Cnvrg
   module Downloader
     OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
+    MAXIMUM_BACKOFF = 64
+    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 20
     attr_accessor :bucket, :client
     class Client
       def initialize(params)
@@ -41,6 +43,35 @@ module Cnvrg
         Cnvrg::Helpers.decrypt(@key, @iv, str)
       end
 
+      def safe_upload(storage_path, local_path)
+        n = 1
+        error = nil
+        while n <= RETRIES
+          begin
+            self.upload(storage_path, local_path)
+            error = nil
+            break
+          rescue => e
+            backoff_time_seconds = backoff_time(n)
+
+            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
+            if n < RETRIES
+              message += ", next retry in: #{backoff_time_seconds} seconds"
+            else
+              message += ", done retry, continuing to the next file"
+            end
+            Cnvrg::Logger.log_error_message(message)
+
+            sleep backoff_time_seconds
+
+            n += 1
+            error = e
+          end
+        end
+        raise error if error.present?
+        true
+      end
+
       def self.factory(params)
         params = params.as_json
         case params["storage"]
@@ -53,6 +84,18 @@ module Cnvrg
           return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
         end
       end
+
+      private
+
+      def random_number_milliseconds
+        rand(1000) / 1000.0
+      end
+
+
+      def backoff_time(n)
+        return [((2**n)+random_number_milliseconds), MAXIMUM_BACKOFF].min
+      end
+
     end
   end
 end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -21,28 +21,12 @@ module Cnvrg
 
         def upload(storage_path, local_path)
           begin
-            try_upload_file(storage_path, local_path)
+            client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
           rescue => e
             raise e
           end
         end
 
-        def try_upload_file(storage_path, local_path)
-          c = 20
-          error = nil
-          while c > 0
-            begin
-              client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
-              break
-            rescue => e
-              c -= 1
-              error = e
-            end
-          end
-          raise error if error.present?
-          return true
-        end
-
         def fetch_files(prefix: nil, marker: nil, limit: 10000)
           blobs = client.list_blobs(@container, prefix: prefix, max_results: limit, marker: marker)
           next_marker = blobs.continuation_token

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -53,7 +53,8 @@ module Cnvrg
           ### storage path is the path inside s3 (after the bucket)
           # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
           o = aws_bucket.object(storage_path)
-          o.upload_file(local_path, @upload_options)
+          success = o.upload_file(local_path, @upload_options)
+          return success
         rescue => e
           raise e
         end

data/lib/cnvrg/files.rb

@@ -880,6 +880,7 @@ module Cnvrg
       end
 
     end
+
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -887,16 +888,14 @@ module Cnvrg
       rescue => e
         return false
       end
+    end
 
-      return true
-
+    def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
+                                                                              username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
+      Cnvrg::CLI.is_response_success(response,false)
+      return response
     end
-      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
-        Cnvrg::CLI.is_response_success(response,false)
-        return response
-      end
 
     def end_commit(commit_sha1,force:false,message:"")
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force,message:message})

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.21'
+  VERSION = '1.6.22'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.21
+  version: 1.6.22
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-05 00:00:00.000000000 Z
+date: 2020-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.