cnvrg 1.6.0.13 → 1.6.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 345b4c6497b1f7621f04a5d5ec4fb7d51d1939da11cb59e70ba5fda466dd8755
-  data.tar.gz: e8321196067449dac95a2e12d2adf74853ebde8f4eda0f0783337a6b5854ea88
+  metadata.gz: 4292d9949e1478ec370c60a31590f1463e8c38729429d457fe4b4c69a400cee7
+  data.tar.gz: f1581daae4274c0ba34615575481f6da1acaf72ba25c86171e1a38bb99105fec
 SHA512:
-  metadata.gz: 7b89ffd39a4a95a583de590188708036ca82cf364c058b7de40b9b95f622daa289f44481a5afa880753f26446495ee5a731a3136441ab2626ecdef33a206e457
-  data.tar.gz: baf172073f91d7d29495d4cb3fcbadef8b396ea7e8b066c11fcd692e01daeb2be9dbb60e9db8fa619b7294ce00be2a9ad1e4fd156ba2f067eacbcf78aa32e1b2
+  metadata.gz: 8652f82423f102949a61725653b4e04380b40c112cee9b152d72be5ce59b3869f796ae04be786f21e682e90c8342b89cd6530139c63ee5e92d8ce675a53ff0a9
+  data.tar.gz: 4858a5d92c12c550d62221bc046c08601ac7fc4b09bd2a2bf28076036655cd8e931fbaf5efd9538d40ffdbffdcb4282a7bcb1c776cb6545cb0408012bafa6260

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '~> 2'
+    spec.add_runtime_dependency 'aws-sdk', '~> 2.11.417'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
     spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'

data/lib/cnvrg/cli.rb

@@ -753,11 +753,14 @@ module Cnvrg
     desc 'data init', 'Init dataset directory', :hide => true
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
-    def init_data(public, bucket: nil)
+    def init_data(public, bucket: nil, title: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
         dataset_name = File.basename(Dir.getwd)
+        if title.present?
+          dataset_name = title
+        end
         if File.directory?(Dir.getwd + "/.cnvrg")
           config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
           log_message("Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED)
@@ -845,7 +848,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        return if check_pod_restart[1] ## It means that all datasets downloaded successfully
         commit = options["commit"] || commit
         only_tree = options["only_tree"] || only_tree
         read = options["read"] || read || false
@@ -1182,7 +1184,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', chunk_size: 1000)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1201,31 +1203,44 @@ module Cnvrg
         end
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
-        response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
-        unless response #means we failed in the start commit.
-          raise SignalException.new(1, "Cant put files into server, check the dataset slug")
+        if commit.blank?
+          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+          unless response #means we failed in the start commit.
+            raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
+          end
+          @commit =  response['result']['commit_sha1']
+        elsif commit.eql? "latest"
+          response = @datafiles.last_valid_commit()
+          unless response #means we failed in the start commit.
+            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
+          end
+          @commit = response['result']['sha1']
+        else
+          @commit = commit
         end
-        @commit = response['result']['commit_sha1']
         #dir shouldnt have starting or ending slash.
         dir = dir[0..-2] if dir.end_with? '/'
         dir = dir[1..-1] if dir.start_with? '/'
 
-        progressbar = ProgressBar.create(:title => "Upload Progress",
-                                         :progress_mark => '=',
-                                         :format => "%b>>%i| %p%% %t",
-                                         :starting_at => 0,
-                                         :total => @files.size,
-                                         :autofinish => true)
         @files.each_slice(chunk_size).each do |list_files|
           temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
           #will throw a signal exception if something goes wrong.
-          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, progressbar: progressbar, prefix: dir)
+          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, prefix: dir, total: @files.size)
         end
-        res = @datafiles.put_commit(@commit)
-        unless res.is_success?
-          raise SignalException.new(1, res.msg)
+        if commit.blank?
+          res = @datafiles.put_commit(@commit)
+          unless res.is_success?
+            raise SignalException.new(1, res.msg)
+          end
+        else
+          res = @datafiles.end_commit(@commit,false, success: true )
+          msg = res['result']
+          response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
+          unless response.is_success?
+            raise SignalException.new(1, res.msg)
+          end
         end
-        log_message("Upload finished Successfully", Thor::Shell::Color::GREEN)
+        log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false
@@ -1781,7 +1796,6 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        return if check_pod_restart[0] ## It means that project downloaded successfully
         url_parts = project_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
         slug = url_parts[project_index + 1]
@@ -1950,12 +1964,11 @@ module Cnvrg
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
       if !force and !init
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
-        invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
+        total_deleted, total_downloaded = invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
-
       # w(new_branch,  verbose,sync,force, tags, chunk_size)
-      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message], :new_branch=>new_branch,
-             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
+      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message, total_deleted: total_deleted, total_downloaded: total_downloaded],
+             :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
 
     end
     desc 'upload_data_new', 'upload_data_new', :hide => true
@@ -1967,31 +1980,47 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
-    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil)
+    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
       begin
-      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
-      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
-      if upload_res < 0
-        return
-      end
-      invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res], :new_branch=>new_branch, :force =>force
-      if tags
-        log_message('Uploading Tags', Thor::Shell::Color::BLUE)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-        begin
-          tag_file = File.open(options[:tags], "r+")
-          status = @dataset.upload_tags_via_yml(tag_file)
-        rescue
-          log_message('Tags file not found', Thor::Shell::Color::RED)
-          return
+        commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
+        files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
+
+        upload_size = files_to_upload + upload_errors.try(:size) rescue 0
+        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload, sync: sync], :new_branch=>new_branch, :force =>force
+        if tags
+          log_message('Uploading Tags', Thor::Shell::Color::BLUE)
+          dataset_dir = is_cnvrg_dir(Dir.pwd)
+          @dataset = Dataset.new(dataset_dir)
+          begin
+            tag_file = File.open(options[:tags], "r+")
+            status = @dataset.upload_tags_via_yml(tag_file)
+          rescue
+            log_message('Tags file not found', Thor::Shell::Color::RED)
+            return
+          end
+          if status
+            log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
+          else
+            log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
+          end
         end
-        if status
-          log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
-        else
-          log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
+        if total_deleted > 0
+          log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
+        end
+
+        if total_downloaded > 0
+          log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
+        end
+        if upload_size > 0
+          log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
+        end
+
+        if upload_errors.try(:size) > 0
+          log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
+          upload_errors.each do |file_hash|
+            log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
+          end
         end
-      end
       rescue => e
         Cnvrg::CLI.log_message(e.message, 'red')
         Cnvrg::Logger.log_error(e)
@@ -2021,17 +2050,16 @@ module Cnvrg
       verify_logged_in(true)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
-      direct = options[:direct]
       new_branch = options["new_branch"] || false
       force = options["force"] || false
       chunk_size = options["chunk_size"] || false
       message = options["message"]
-      commit_sha1 = nil
       @dataset = Dataset.new(dataset_dir)
       @dataset.backup_idx
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       next_commit = @dataset.get_next_commit #if there was a partial commit..
-      chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
+      files_list = @dataset.list_all_files
+      chunks = (files_list.length.to_f / chunk_size).ceil
       resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset, message: message)
       if !resp['result']['can_commit']
         log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
@@ -2042,7 +2070,7 @@ module Cnvrg
         @dataset.set_partial_commit(next_commit)
       end
       @dataset.set_next_commit(commit_sha1)
-      return commit_sha1
+      return commit_sha1, files_list
     end
 
 
@@ -2050,7 +2078,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
-    def end_commit_data(commit, success: true, uploaded_files: 0)
+    def end_commit_data(commit, success: true, uploaded_files: 0, sync: false)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2065,7 +2093,12 @@ module Cnvrg
             @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+            if sync
+              message = "#{check} Data sync finished"
+            else
+              message = "#{check} Data upload finished"
+            end
+            log_message(message, Thor::Shell::Color::GREEN)
             @dataset.remove_next_commit #takes the next commit and put it as current commit
             @dataset.set_partial_commit(nil)
             @dataset.backup_idx
@@ -2118,54 +2151,53 @@ module Cnvrg
     # method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
-    def upload_data_files(new_commit, *files)
+    def upload_data_files(new_commit, files_list: [])
       begin
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
-      new_commit ||= @dataset.get_next_commit
-      partial_commit = @dataset.get_partial_commit
-      if new_commit.blank?
-        log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
-        return false
-      end
-      force = options[:force] || false
-      chunk_size = options[:chunk_size]
-      chunk_size = [chunk_size, 1].max
-      new_branch = options["new_branch"] || false
-      new_tree = {}
-      force = options["force"] || false
-      parallel_threads = options["parallel"] || ParallelThreads
-      all_files = @dataset.list_all_files
-      progressbar = ProgressBar.create(:title => "Upload Progress",
-                                       :progress_mark => '=',
-                                       :format => "%b>>%i| %p%% %t",
-                                       :starting_at => 0,
-                                       :total => all_files.length,
-                                       :autofinish => true)
-      files_uploaded = 0
-      all_files.each_slice(chunk_size).each do |list_files|
-        Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
-        temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
-        upload_resp = @files.upload_multiple_files(new_commit, temp_tree,
-                                                   threads: parallel_threads,
-                                                   force: force,
-                                                   new_branch: new_branch,
-                                                   progressbar: progressbar,
-                                                   partial_commit: partial_commit)
-        files_uploaded += upload_resp
-        temp_tree.each do |k, v|
-          new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
-        end
-      end
-      @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
+        new_commit ||= @dataset.get_next_commit
+        partial_commit = @dataset.get_partial_commit
+        if new_commit.blank?
+          log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
+          return false
+        end
+        chunk_size = options[:chunk_size]
+        chunk_size = [chunk_size, 1].max
+        new_branch = options["new_branch"] || false
+        new_tree = {}
+        force = options["force"] || false
+        parallel_threads = options["parallel"] || ParallelThreads
+        all_files = files_list
+        all_files = @dataset.list_all_files if files_list.blank?
+        files_uploaded = 0
+        upload_errors = []
+
+        all_files.each_slice(chunk_size).each do |list_files|
+          Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
+          temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+          upload_resp, upload_error_files = @files.upload_multiple_files(new_commit, temp_tree,
+                                                                         threads: parallel_threads,
+                                                                         force: force,
+                                                                         new_branch: new_branch,
+                                                                         partial_commit: partial_commit,
+                                                                         total: all_files.length)
+
+          files_uploaded += upload_resp
+          upload_errors += upload_error_files if upload_error_files.present?
+          temp_tree.each do |k, v|
+            new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
+          end
+        end
+
+        @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
       rescue => e
         Cnvrg::Logger.log_error(e)
         raise e
       end
-      return files_uploaded
+      return files_uploaded, upload_errors.try(:flatten).try(:compact)
     end
 
 
@@ -2293,7 +2325,9 @@ module Cnvrg
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
         @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
+
         @files.delete_files_from_server(deleted, commit_sha1)
+
         progressbar.finish
         res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
         unless Cnvrg::CLI.is_response_success(res, false)
@@ -2431,52 +2465,51 @@ module Cnvrg
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
-        all_files = all_files
-        @dataset.generate_idx
         res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
         unless CLI.is_response_success(res, false)
           log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
           exit(1)
         end
-        result = res["result"]["tree"]
-        commit = res["result"]["commit"]
-        can_commit = res["result"]["can_commit"] || false #can commit means that our next commit is newer than the latest commit
-        # so if can_commit is true it means that we have to be up-to-date with the dataset.
-        update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
+        result = res["result"]
+        tree = result["tree"]
+        commit = result["commit"]
+        update_total = [tree['added'], tree["updated_on_server"], tree["conflicts"], tree["deleted"]].compact.flatten.size
         successful_changes  = 0
-        if update_total == 0 or can_commit
-          log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
-          return true
-        elsif options["verbose"]
-          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
+        if update_total == 0
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
+          return 0, 0
         else
+          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE, options["verbose"])
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
         end
-        Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
-        Cnvrg::Logger.log_info("Compare idx res: #{result}")
+        Cnvrg::Logger.log_info("Current commit: #{@dataset.last_local_commit}, destination commit: #{commit}")
+        Cnvrg::Logger.log_info("Compare idx res: #{tree}")
         progressbar = ProgressBar.create(:title => "Download Progress",
-                                           :progress_mark => '=',
-                                           :format => "%b>>%i| %p%% %t",
-                                           :starting_at => 0,
-                                           :total => update_total,
-                                           :autofinish => true)
-        conflicts = @files.mark_conflicts(result)
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => update_total,
+                                         :autofinish => true)
 
+        conflicts = @files.mark_conflicts(tree)
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
-        update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
-        added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
-        # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
-        deleted = result["deleted"].to_a
+        update_res = @files.download_files_in_chunks(tree["updated_on_server"], progress: progressbar) if tree["updated_on_server"].present?
+        added_res = @files.download_files_in_chunks(tree["added"], progress: progressbar) if tree["added"].present?
+        deleted = tree["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
-        progressbar.progress += deleted.size if progressbar.present?
+
         if !delete_res
           log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
           exit(1)
         end
+
+        progressbar.progress += deleted.size if progressbar.present? and deleted.size > 0
+
         success = (update_res.blank? or update_res.is_success?)
         success &= (delete_res.blank? or delete_res.is_success?)
         success &= (added_res.blank? or added_res.is_success?)
+
         if success
           # update idx with latest commit
           @dataset.update_idx_with_commit!(commit)
@@ -2486,17 +2519,24 @@ module Cnvrg
             log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
             log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
+            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, !sync)
           end
-          return true
+
+          total_deleted = deleted.try(:size)
+          total_downloaded = tree["added"].try(:size) || 0
+          total_downloaded +=  tree["updated_on_server"].try(:size) if tree["updated_on_server"].present?
+
+          return total_deleted, total_downloaded
+        else
+          return []
         end
       rescue SignalException => e
         Cnvrg::Logger.log_error(e)
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       rescue => e
-        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         Cnvrg::Logger.log_error(e)
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
         exit(1)
       end
     end
@@ -2854,7 +2894,7 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
-    def  sync(direct = true)
+    def sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
       log_start(__method__, args, options)
@@ -2898,7 +2938,7 @@ module Cnvrg
     method_option :image, :type => :string, :aliases => ["--image"], :default => nil
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true, \"use_cached\": true]'", :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -3382,10 +3422,6 @@ module Cnvrg
           output_dir  = "output"
         end
         image = options["image"] || nil
-        if image.blank?
-          image  = "cnvrg"
-        end
-
         forced_commit = nil
         if sync_before and !project.is_git
           if force
@@ -5487,7 +5523,7 @@ module Cnvrg
         end
       end
 
-      def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
+      def self.log_message(message, type = Thor::Shell::Color::BLUE)
         if $LOG.blank?
           ### handle case when $LOG is not initialized
           CLI.new.log_handler
@@ -5509,11 +5545,12 @@ module Cnvrg
           color = nil
           $LOG.info message: message, type: "unknown"
         end
-        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}" if to_print
+        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}"
       end
 
       def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
-        CLI.log_message(message, type, to_print: to_print)
+        return if not to_print
+        CLI.log_message(message, type)
       end
 
       def log_error(e)

data/lib/cnvrg/data.rb

@@ -22,11 +22,14 @@ module Cnvrg
     desc "data init", "Set current directory as dataset directory"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
+
     def init
       cli = Cnvrg::CLI.new()
       public = options["public"]
       bucket = options["bucket"]
-      cli.init_data(public, bucket: bucket)
+      title = options["title"]
+      cli.init_data(public, bucket: bucket, title: title)
     end
 
     desc "data link DATASET_SLUG", "Set current directory as dataset directory"
@@ -182,10 +185,12 @@ module Cnvrg
 
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
-      cli.data_put(dataset_url, files: files, dir: dir)
+      commit = options[:commit]
+      cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -114,54 +114,82 @@ module Cnvrg
     end
 
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
         Cnvrg::Logger.log_info("Sending Upload Files request")
-        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
-        unless Cnvrg::CLI.is_response_success(upload_resp, false)
+        error = nil
+        upload_resp = nil
+        10.times do
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
+          if Cnvrg::CLI.is_response_success(upload_resp, false)
+            error = nil
+            break
+          end
+          error = upload_resp
           Cnvrg::Logger.log_method(bind: binding)
-          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          Cnvrg::Logger.log_info("Got an error message from server, #{upload_resp.try(:fetch, "message")}, trying again")
         end
+        raise Exception.new("Can't upload data files: #{error["message"]}") if error.present?
+
         Cnvrg::Logger.log_info("Uploading files")
         results = upload_resp['result'].with_indifferent_access
+
         if results['files'].blank?
-          progressbar.progress += tree.keys.length if progressbar.present?
-          return 0
+          return 0, []
+        end
+
+        if @temp_upload_progressbar.blank?
+          @temp_upload_progressbar = ProgressBar.create(:title => "Upload Progress",
+                                                        :progress_mark => '=',
+                                                        :format => "%b>>%i| %p%% %t",
+                                                        :starting_at => 0,
+                                                        :total => total,
+                                                        :autofinish => true)
         end
+
         files = results['files']
-        progressbar.progress += tree.keys.length - files.length if progressbar.present?
-        Parallel.map((files.keys), {in_threads: threads}) do |k|
+        upload_error_files = []
+        @temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
+        Parallel.map((files.keys), in_threads: threads) do |k|
           o = tree[k].merge(files[k])
-          upload_single_file(o)
-          progressbar.progress += 1 if progressbar.present?
+          success = upload_single_file(o)
+          if not success
+            upload_error_files << {absolute_path: o[:absolute_path]}
+            files.except!(k)
+            tree.except!(k)
+            Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
+          end
+          @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
         end
         blob_ids = files.values.map {|f| f['bv_id']}
-        dirs = tree.keys.select {|k| tree[k].nil?} || []
-        Cnvrg::Logger.info("Sending Upload files save")
-        upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-        unless Cnvrg::CLI.is_response_success(upload_resp, false)
-          Cnvrg::Logger.log_method(bind: binding)
-          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        if blob_ids.present?
+          dirs = tree.keys.select {|k| tree[k].nil?} || []
+          Cnvrg::Logger.info("Sending Upload files save")
+          upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+          unless Cnvrg::CLI.is_response_success(upload_resp, false)
+            Cnvrg::Logger.log_method(bind: binding)
+            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          end
         end
         Cnvrg::Logger.log_info("Upload Success")
-        return files.keys.length
+        return files.try(:keys).try(:length), upload_error_files
       rescue => e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
       end
-
     end
 
     def upload_single_file(file)
       begin
-      file = file.as_json
-      Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
-      @downloader.upload(file["path"], file["absolute_path"])
-      Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
+        file = file.as_json
+        Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
+        @downloader.safe_upload(file["path"], file["absolute_path"])
+        Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
+        return false
       end
     end
 
@@ -802,6 +830,7 @@ module Cnvrg
       end
       return true
     end
+
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -809,9 +838,6 @@ module Cnvrg
       rescue => e
         return Cnvrg::Result.new(false, '')
       end
-
-      return Cnvrg::Result.new(true, '')
-
     end
 
     def download_dir(dataset_home, absolute_path)
@@ -856,6 +882,7 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
+
     def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -863,14 +890,24 @@ module Cnvrg
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
-      Cnvrg::CLI.is_response_success(response, true)
-      return response
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
+                                                                                username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
+      rescue => e
+        return false
+      end
+    end
+    def last_valid_commit()
+      begin
+        #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
+        # create a new commit.
+        response = Cnvrg::API.request("#{base_resource}/last_valid_commit", 'GET')
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
       rescue => e
         return false
       end
-
     end
 
     def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )

data/lib/cnvrg/dataset.rb

@@ -103,8 +103,12 @@ module Cnvrg
 
     def backup_idx
       Cnvrg::Logger.log_info("Backup idx")
-      idx = self.get_idx
-      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
+      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+        FileUtils.cp "#{self.local_path}/.cnvrg/idx.yml", "#{self.local_path}/.cnvrg/idx.yml.backup"
+      else
+        idx = {commit: nil, tree: {}}
+        File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
+      end
     end
 
     def restore_idx
@@ -780,14 +784,6 @@ module Cnvrg
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
     end
 
-    def get_current_commit()
-      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
-        return nil
-      end
-      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-      return idx_hash[:commit]
-    end
-
     def compare_commit(commit)
       if commit.nil? or commit.empty?
         commit = last_local_commit

data/lib/cnvrg/downloader/client.rb

@@ -2,6 +2,8 @@
 module Cnvrg
   module Downloader
     OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
+    MAXIMUM_BACKOFF = 64
+    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 20
     attr_accessor :bucket, :client
     class Client
       def initialize(params)
@@ -41,6 +43,35 @@ module Cnvrg
         Cnvrg::Helpers.decrypt(@key, @iv, str)
       end
 
+      def safe_upload(storage_path, local_path)
+        n = 1
+        error = nil
+        while n <= RETRIES
+          begin
+            self.upload(storage_path, local_path)
+            error = nil
+            break
+          rescue => e
+            backoff_time_seconds = backoff_time(n)
+
+            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
+            if n < RETRIES
+              message += ", next retry in: #{backoff_time_seconds} seconds"
+            else
+              message += ", done retry, continuing to the next file"
+            end
+            Cnvrg::Logger.log_error_message(message)
+
+            sleep backoff_time_seconds
+
+            n += 1
+            error = e
+          end
+        end
+        raise error if error.present?
+        true
+      end
+
       def self.factory(params)
         params = params.as_json
         case params["storage"]
@@ -53,6 +84,18 @@ module Cnvrg
           return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
         end
       end
+
+      private
+
+      def random_number_milliseconds
+        rand(1000) / 1000.0
+      end
+
+
+      def backoff_time(n)
+        return [((2**n)+random_number_milliseconds), MAXIMUM_BACKOFF].min
+      end
+
     end
   end
 end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -20,7 +20,11 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
+          begin
+            client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
+          rescue => e
+            raise e
+          end
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 10000)

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -38,7 +38,11 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          @bucket.create_file(local_path, storage_path)
+          begin
+            @bucket.create_file(local_path, storage_path)
+          rescue => e
+            raise e
+          end
         end
       end
     end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -53,7 +53,10 @@ module Cnvrg
           ### storage path is the path inside s3 (after the bucket)
           # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
           o = aws_bucket.object(storage_path)
-          o.upload_file(local_path, @upload_options)
+          success = o.upload_file(local_path, @upload_options)
+          return success
+        rescue => e
+          raise e
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 1000)

data/lib/cnvrg/files.rb

@@ -6,6 +6,7 @@ require 'net/http'
 require 'cnvrg/result'
 module Cnvrg
   class Files
+    ParallelThreads = Cnvrg::Helpers.parallel_threads
     VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
@@ -85,6 +86,7 @@ module Cnvrg
         Cnvrg::Logger.log_info("Upload files to older server..")
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
+
       files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
       resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
@@ -95,10 +97,11 @@ module Cnvrg
       files = res['files']
 
       #upload files
-      blob_ids = Parallel.map(files.keys, self.get_upload_options) do |file|
+      blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
+
           Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
           Cnvrg::Logger.log_error(e)
           Cnvrg::Logger.log_method(bind: binding)
@@ -877,6 +880,7 @@ module Cnvrg
       end
 
     end
+
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -884,16 +888,14 @@ module Cnvrg
       rescue => e
         return false
       end
+    end
 
-      return true
-
+    def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
+                                                                              username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
+      Cnvrg::CLI.is_response_success(response,false)
+      return response
     end
-      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
-        Cnvrg::CLI.is_response_success(response,false)
-        return response
-      end
 
     def end_commit(commit_sha1,force:false,message:"")
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force,message:message})

data/lib/cnvrg/project.rb

@@ -329,10 +329,13 @@ module Cnvrg
     def get_storage_client
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
       if Cnvrg::CLI.is_response_success(response, false)
+
         client_params = response['client']
       else
+
         client_params = get_storage_client_fallback
       end
+
       Cnvrg::Downloader::Client.factory(client_params)
     end
 
@@ -429,13 +432,9 @@ module Cnvrg
       end
       list_ignore_new = list_ignore.map{|x| x.gsub("//","/")} rescue []
       # list.each do |e|
-      project_root  = Pathname.new(self.local_path)
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
-        relative_path = Pathname.new(File.expand_path(e))
-        label = relative_path.relative_path_from(project_root)
-        if not Cnvrg::Files.valid_file_name?(label)
-          raise StandardError.new("#{label} is not a valid file name")
-        end
+        label = e.sub(self.local_path + "/", "")
+
         if list_ignore_new.include? label
           next
         end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.6.0.13'
+  VERSION = '1.6.3'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.0.13
+  version: 1.6.3
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-21 00:00:00.000000000 Z
+date: 2020-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -210,14 +210,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.11.417
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2'
+        version: 2.11.417
 - !ruby/object:Gem::Dependency
   name: signet
   requirement: !ruby/object:Gem::Requirement