cnvrg 1.6.3.1 → 1.6.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 031b5d643980f087c06ded03bb010a85b54127fdf4a95bcc90cd8d75211bb7a0
-  data.tar.gz: 8c41307f0535e9ce926ee272f2806e9fb2444a09ddc3aeb55cf2eb5265016216
+  metadata.gz: cd6065b7fe98eebf485fe2813928c509d9f23700c0e6e34f6cbf880f6717e39b
+  data.tar.gz: 9bc650433cd77fba6de2edd9b15ad380c9302dc9eb250c35ffa7b2e3c9668e3f
 SHA512:
-  metadata.gz: bf9afdf0369891dadd4c6dc1bdcd22883d688fafe68be8ca70927fe521576980eb3563a4d57eab9c5aa7f6d8d16d619a7631f8e8d8da513901b47eb72d95ae8c
-  data.tar.gz: 61ef3ef8107e2c7589ccc17b7e434e4cdde997490b17adb6d2bf32e9876d9f8bdb039c6f269cc391e6cce2d16ce9adce052deded77880b564a4bd7098031f48f
+  metadata.gz: a175f7aaeda31208f381beeefd924bad22e42421d3da023e14e6468460cbd9b3a0fc36602fd4113f7e023ee81877c9cc9f0e659d18e539a774fcc5dc292038aa
+  data.tar.gz: 3dfcbf2f22735ad612cea0974ba75709742371737fc3bfd8a6c6b84c79acdcca30e736b21e39cd397b7394f59891cb649f30ac58ab3ae1be426e1c9a376aa1e8

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '~> 2.11.417'
+    spec.add_runtime_dependency 'aws-sdk', '~> 2'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
     spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'

data/lib/cnvrg/cli.rb

@@ -753,14 +753,11 @@ module Cnvrg
     desc 'data init', 'Init dataset directory', :hide => true
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
-    def init_data(public, bucket: nil, title: nil)
+    def init_data(public, bucket: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
         dataset_name = File.basename(Dir.getwd)
-        if title.present?
-          dataset_name = title
-        end
         if File.directory?(Dir.getwd + "/.cnvrg")
           config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
           log_message("Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED)
@@ -848,6 +845,7 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
+        return if check_pod_restart[1] ## It means that all datasets downloaded successfully
         commit = options["commit"] || commit
         only_tree = options["only_tree"] || only_tree
         read = options["read"] || read || false
@@ -1184,7 +1182,7 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
+    def data_put(dataset_url, files: [], dir: '', chunk_size: 1000)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -1203,44 +1201,31 @@ module Cnvrg
         end
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
-        if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
-          unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
-          end
-          @commit =  response['result']['commit_sha1']
-        elsif commit.eql? "latest"
-          response = @datafiles.last_valid_commit()
-          unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
-          end
-          @commit = response['result']['sha1']
-        else
-          @commit = commit
+        response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+        unless response #means we failed in the start commit.
+          raise SignalException.new(1, "Cant put files into server, check the dataset slug")
         end
+        @commit = response['result']['commit_sha1']
         #dir shouldnt have starting or ending slash.
         dir = dir[0..-2] if dir.end_with? '/'
         dir = dir[1..-1] if dir.start_with? '/'
 
+        progressbar = ProgressBar.create(:title => "Upload Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => @files.size,
+                                         :autofinish => true)
         @files.each_slice(chunk_size).each do |list_files|
           temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
           #will throw a signal exception if something goes wrong.
-          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, prefix: dir, total: @files.size)
+          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, progressbar: progressbar, prefix: dir)
         end
-        if commit.blank?
-          res = @datafiles.put_commit(@commit)
-          unless res.is_success?
-            raise SignalException.new(1, res.msg)
-          end
-        else
-          res = @datafiles.end_commit(@commit,false, success: true )
-          msg = res['result']
-          response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
-          unless response.is_success?
-            raise SignalException.new(1, res.msg)
-          end
+        res = @datafiles.put_commit(@commit)
+        unless res.is_success?
+          raise SignalException.new(1, res.msg)
         end
-        log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
+        log_message("Upload finished Successfully", Thor::Shell::Color::GREEN)
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false
@@ -1796,6 +1781,7 @@ module Cnvrg
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
+        return if check_pod_restart[0] ## It means that project downloaded successfully
         url_parts = project_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
         slug = url_parts[project_index + 1]
@@ -1964,11 +1950,12 @@ module Cnvrg
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
       if !force and !init
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
-        total_deleted, total_downloaded = invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
+        invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
+
       # w(new_branch,  verbose,sync,force, tags, chunk_size)
-      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message, total_deleted: total_deleted, total_downloaded: total_downloaded],
-             :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
+      invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message], :new_branch=>new_branch,
+             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
 
     end
     desc 'upload_data_new', 'upload_data_new', :hide => true
@@ -1980,47 +1967,31 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
 
-    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil, total_deleted: 0, total_downloaded: 0)
+    def upload_data_new(new_branch, verbose, sync, force, tags, chunk_size, message:nil)
       begin
-        commit, files_list = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
-        files_to_upload, upload_errors = invoke :upload_data_files,[commit, files_list: files_list],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
-
-        upload_size = files_to_upload + upload_errors.try(:size) rescue 0
-        invoke :end_commit_data,[commit, success: true, uploaded_files: files_to_upload, sync: sync], :new_branch=>new_branch, :force =>force
-        if tags
-          log_message('Uploading Tags', Thor::Shell::Color::BLUE)
-          dataset_dir = is_cnvrg_dir(Dir.pwd)
-          @dataset = Dataset.new(dataset_dir)
-          begin
-            tag_file = File.open(options[:tags], "r+")
-            status = @dataset.upload_tags_via_yml(tag_file)
-          rescue
-            log_message('Tags file not found', Thor::Shell::Color::RED)
-            return
-          end
-          if status
-            log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
-          else
-            log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
-          end
-        end
-        if total_deleted > 0
-          log_message("#{total_deleted} files deleted successfully.", Thor::Shell::Color::GREEN)
-        end
-
-        if total_downloaded > 0
-          log_message("#{total_downloaded} files downloaded successfully.", Thor::Shell::Color::GREEN)
-        end
-        if upload_size > 0
-          log_message("#{files_to_upload}/#{upload_size} files uploaded successfully.", Thor::Shell::Color::GREEN)
+      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size, :message => message
+      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
+      if upload_res < 0
+        return
+      end
+      invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res], :new_branch=>new_branch, :force =>force
+      if tags
+        log_message('Uploading Tags', Thor::Shell::Color::BLUE)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        begin
+          tag_file = File.open(options[:tags], "r+")
+          status = @dataset.upload_tags_via_yml(tag_file)
+        rescue
+          log_message('Tags file not found', Thor::Shell::Color::RED)
+          return
         end
-
-        if upload_errors.try(:size) > 0
-          log_message("#{upload_errors.try(:size)}/#{upload_size} files didn't upload:", Thor::Shell::Color::RED)
-          upload_errors.each do |file_hash|
-            log_message("#{file_hash[:absolute_path]}", Thor::Shell::Color::RED)
-          end
+        if status
+          log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
+        else
+          log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
         end
+      end
       rescue => e
         Cnvrg::CLI.log_message(e.message, 'red')
         Cnvrg::Logger.log_error(e)
@@ -2050,16 +2021,17 @@ module Cnvrg
       verify_logged_in(true)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
+      direct = options[:direct]
       new_branch = options["new_branch"] || false
       force = options["force"] || false
       chunk_size = options["chunk_size"] || false
       message = options["message"]
+      commit_sha1 = nil
       @dataset = Dataset.new(dataset_dir)
       @dataset.backup_idx
       @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
       next_commit = @dataset.get_next_commit #if there was a partial commit..
-      files_list = @dataset.list_all_files
-      chunks = (files_list.length.to_f / chunk_size).ceil
+      chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
       resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset, message: message)
       if !resp['result']['can_commit']
         log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
@@ -2070,7 +2042,7 @@ module Cnvrg
         @dataset.set_partial_commit(next_commit)
       end
       @dataset.set_next_commit(commit_sha1)
-      return commit_sha1, files_list
+      return commit_sha1
     end
 
 
@@ -2078,7 +2050,7 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
-    def end_commit_data(commit, success: true, uploaded_files: 0, sync: false)
+    def end_commit_data(commit, success: true, uploaded_files: 0)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2093,12 +2065,7 @@ module Cnvrg
             @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
-            if sync
-              message = "#{check} Data sync finished"
-            else
-              message = "#{check} Data upload finished"
-            end
-            log_message(message, Thor::Shell::Color::GREEN)
+            log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
             @dataset.remove_next_commit #takes the next commit and put it as current commit
             @dataset.set_partial_commit(nil)
             @dataset.backup_idx
@@ -2151,53 +2118,54 @@ module Cnvrg
     # method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
-    def upload_data_files(new_commit, files_list: [])
+    def upload_data_files(new_commit, *files)
       begin
-        verify_logged_in(true)
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
-        new_commit ||= @dataset.get_next_commit
-        partial_commit = @dataset.get_partial_commit
-        if new_commit.blank?
-          log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
-          return false
-        end
-        chunk_size = options[:chunk_size]
-        chunk_size = [chunk_size, 1].max
-        new_branch = options["new_branch"] || false
-        new_tree = {}
-        force = options["force"] || false
-        parallel_threads = options["parallel"] || ParallelThreads
-        all_files = files_list
-        all_files = @dataset.list_all_files if files_list.blank?
-        files_uploaded = 0
-        upload_errors = []
-
-        all_files.each_slice(chunk_size).each do |list_files|
-          Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
-          temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
-          upload_resp, upload_error_files = @files.upload_multiple_files(new_commit, temp_tree,
-                                                                         threads: parallel_threads,
-                                                                         force: force,
-                                                                         new_branch: new_branch,
-                                                                         partial_commit: partial_commit,
-                                                                         total: all_files.length)
-
-          files_uploaded += upload_resp
-          upload_errors += upload_error_files if upload_error_files.present?
-          temp_tree.each do |k, v|
-            new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
-          end
-        end
-
-        @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
+      new_commit ||= @dataset.get_next_commit
+      partial_commit = @dataset.get_partial_commit
+      if new_commit.blank?
+        log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
+        return false
+      end
+      force = options[:force] || false
+      chunk_size = options[:chunk_size]
+      chunk_size = [chunk_size, 1].max
+      new_branch = options["new_branch"] || false
+      new_tree = {}
+      force = options["force"] || false
+      parallel_threads = options["parallel"] || ParallelThreads
+      all_files = @dataset.list_all_files
+      progressbar = ProgressBar.create(:title => "Upload Progress",
+                                       :progress_mark => '=',
+                                       :format => "%b>>%i| %p%% %t",
+                                       :starting_at => 0,
+                                       :total => all_files.length,
+                                       :autofinish => true)
+      files_uploaded = 0
+      all_files.each_slice(chunk_size).each do |list_files|
+        Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
+        temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+        upload_resp = @files.upload_multiple_files(new_commit, temp_tree,
+                                                   threads: parallel_threads,
+                                                   force: force,
+                                                   new_branch: new_branch,
+                                                   progressbar: progressbar,
+                                                   partial_commit: partial_commit)
+        files_uploaded += upload_resp
+        temp_tree.each do |k, v|
+          new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
+        end
+      end
+      @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
       rescue => e
         Cnvrg::Logger.log_error(e)
         raise e
       end
-      return files_uploaded, upload_errors.try(:flatten).try(:compact)
+      return files_uploaded
     end
 
 
@@ -2325,9 +2293,7 @@ module Cnvrg
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
         @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
-
         @files.delete_files_from_server(deleted, commit_sha1)
-
         progressbar.finish
         res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
         unless Cnvrg::CLI.is_response_success(res, false)
@@ -2465,51 +2431,52 @@ module Cnvrg
         dataset_dir = is_cnvrg_dir(Dir.pwd)
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug, dataset: @dataset)
+        all_files = all_files
+        @dataset.generate_idx
         res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
         unless CLI.is_response_success(res, false)
           log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
           exit(1)
         end
-        result = res["result"]
-        tree = result["tree"]
-        commit = result["commit"]
-        update_total = [tree['added'], tree["updated_on_server"], tree["conflicts"], tree["deleted"]].compact.flatten.size
+        result = res["result"]["tree"]
+        commit = res["result"]["commit"]
+        can_commit = res["result"]["can_commit"] || false #can commit means that our next commit is newer than the latest commit
+        # so if can_commit is true it means that we have to be up-to-date with the dataset.
+        update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
         successful_changes  = 0
-        if update_total == 0
-          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
-          return 0, 0
+        if update_total == 0 or can_commit
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
+          return true
+        elsif options["verbose"]
+          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
         else
-          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE, options["verbose"])
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
         end
-        Cnvrg::Logger.log_info("Current commit: #{@dataset.last_local_commit}, destination commit: #{commit}")
-        Cnvrg::Logger.log_info("Compare idx res: #{tree}")
+        Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
+        Cnvrg::Logger.log_info("Compare idx res: #{result}")
         progressbar = ProgressBar.create(:title => "Download Progress",
-                                         :progress_mark => '=',
-                                         :format => "%b>>%i| %p%% %t",
-                                         :starting_at => 0,
-                                         :total => update_total,
-                                         :autofinish => true)
+                                           :progress_mark => '=',
+                                           :format => "%b>>%i| %p%% %t",
+                                           :starting_at => 0,
+                                           :total => update_total,
+                                           :autofinish => true)
+        conflicts = @files.mark_conflicts(result)
 
-        conflicts = @files.mark_conflicts(tree)
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
-        update_res = @files.download_files_in_chunks(tree["updated_on_server"], progress: progressbar) if tree["updated_on_server"].present?
-        added_res = @files.download_files_in_chunks(tree["added"], progress: progressbar) if tree["added"].present?
-        deleted = tree["deleted"].to_a
+        update_res = @files.download_files_in_chunks(result["updated_on_server"], progress: progressbar) if result["updated_on_server"].present?
+        added_res = @files.download_files_in_chunks(result["added"], progress: progressbar) if result["added"].present?
+        # conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
+        deleted = result["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
-
+        progressbar.progress += deleted.size if progressbar.present?
         if !delete_res
           log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
           exit(1)
         end
-
-        progressbar.progress += deleted.size if progressbar.present? and deleted.size > 0
-
         success = (update_res.blank? or update_res.is_success?)
         success &= (delete_res.blank? or delete_res.is_success?)
         success &= (added_res.blank? or added_res.is_success?)
-
         if success
           # update idx with latest commit
           @dataset.update_idx_with_commit!(commit)
@@ -2519,24 +2486,17 @@ module Cnvrg
             log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
             log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
           else
-            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, !sync)
+            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
-
-          total_deleted = deleted.try(:size)
-          total_downloaded = tree["added"].try(:size) || 0
-          total_downloaded +=  tree["updated_on_server"].try(:size) if tree["updated_on_server"].present?
-
-          return total_deleted, total_downloaded
-        else
-          return []
+          return true
         end
       rescue SignalException => e
         Cnvrg::Logger.log_error(e)
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       rescue => e
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         Cnvrg::Logger.log_error(e)
-        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
         exit(1)
       end
     end
@@ -2894,15 +2854,14 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
-    def sync(direct = true)
+    def  sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
       log_start(__method__, args, options)
       log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
       log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
-      job_slug = options['job_slug'] || ENV['CNVRG_JOB_ID']
-      job_type = options['job_type'] || ENV['CNVRG_JOB_TYPE']
-      is_git = ENV['CNVRG_GIT_PROJECT'] == "true" || @project.is_git
+      job_slug = options['job_slug']
+      job_type = options['job_type']
       in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
       in_exp = false if job_type.present? and job_type == "NotebookSession"
       run_download = true
@@ -2938,7 +2897,7 @@ module Cnvrg
     method_option :image, :type => :string, :aliases => ["--image"], :default => nil
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true, \"use_cached\": true]'", :default => ""
+    method_option :datasets, :type => :string, :aliases => ["--datasets"], :desc => "'[{\"id\": \"dataset id\", \"commit\": \"commit id\", \"query\": \"query name\", \"tree_only\": true]'", :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -3422,6 +3381,10 @@ module Cnvrg
           output_dir  = "output"
         end
         image = options["image"] || nil
+        if image.blank?
+          image  = "cnvrg"
+        end
+
         forced_commit = nil
         if sync_before and !project.is_git
           if force
@@ -5523,7 +5486,7 @@ module Cnvrg
         end
       end
 
-      def self.log_message(message, type = Thor::Shell::Color::BLUE)
+      def self.log_message(message, type = Thor::Shell::Color::BLUE, to_print: true)
         if $LOG.blank?
           ### handle case when $LOG is not initialized
           CLI.new.log_handler
@@ -5545,12 +5508,11 @@ module Cnvrg
           color = nil
           $LOG.info message: message, type: "unknown"
         end
-        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}"
+        say "#{color}#{message}#{Thor::Shell::Color::CLEAR}" if to_print
       end
 
       def log_message(message, type=Thor::Shell::Color::GREEN, to_print = true)
-        return if not to_print
-        CLI.log_message(message, type)
+        CLI.log_message(message, type, to_print: to_print)
       end
 
       def log_error(e)

data/lib/cnvrg/data.rb

@@ -22,14 +22,11 @@ module Cnvrg
     desc "data init", "Set current directory as dataset directory"
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
     method_option :bucket, :type => :string, :aliases => ["-b", "--bucket"], :default => ""
-    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
-
     def init
       cli = Cnvrg::CLI.new()
       public = options["public"]
       bucket = options["bucket"]
-      title = options["title"]
-      cli.init_data(public, bucket: bucket, title: title)
+      cli.init_data(public, bucket: bucket)
     end
 
     desc "data link DATASET_SLUG", "Set current directory as dataset directory"
@@ -185,12 +182,10 @@ module Cnvrg
 
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
-      commit = options[:commit]
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
+      cli.data_put(dataset_url, files: files, dir: dir)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -114,82 +114,54 @@ module Cnvrg
     end
 
 
-    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
+    def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
       begin
         Cnvrg::Logger.log_info("Sending Upload Files request")
-        error = nil
-        upload_resp = nil
-        10.times do
-          upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
-          if Cnvrg::CLI.is_response_success(upload_resp, false)
-            error = nil
-            break
-          end
-          error = upload_resp
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
+        unless Cnvrg::CLI.is_response_success(upload_resp, false)
           Cnvrg::Logger.log_method(bind: binding)
-          Cnvrg::Logger.log_info("Got an error message from server, #{upload_resp.try(:fetch, "message")}, trying again")
+          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
         end
-        raise Exception.new("Can't upload data files: #{error["message"]}") if error.present?
-
         Cnvrg::Logger.log_info("Uploading files")
         results = upload_resp['result'].with_indifferent_access
-
         if results['files'].blank?
-          return 0, []
-        end
-
-        if @temp_upload_progressbar.blank?
-          @temp_upload_progressbar = ProgressBar.create(:title => "Upload Progress",
-                                                        :progress_mark => '=',
-                                                        :format => "%b>>%i| %p%% %t",
-                                                        :starting_at => 0,
-                                                        :total => total,
-                                                        :autofinish => true)
+          progressbar.progress += tree.keys.length if progressbar.present?
+          return 0
         end
-
         files = results['files']
-        upload_error_files = []
-        @temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
-        Parallel.map((files.keys), in_threads: threads) do |k|
+        progressbar.progress += tree.keys.length - files.length if progressbar.present?
+        Parallel.map((files.keys), {in_threads: threads}) do |k|
           o = tree[k].merge(files[k])
-          success = upload_single_file(o)
-          if not success
-            upload_error_files << {absolute_path: o[:absolute_path]}
-            files.except!(k)
-            tree.except!(k)
-            Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
-          end
-          @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
+          upload_single_file(o)
+          progressbar.progress += 1 if progressbar.present?
         end
         blob_ids = files.values.map {|f| f['bv_id']}
-        if blob_ids.present?
-          dirs = tree.keys.select {|k| tree[k].nil?} || []
-          Cnvrg::Logger.info("Sending Upload files save")
-          upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-          unless Cnvrg::CLI.is_response_success(upload_resp, false)
-            Cnvrg::Logger.log_method(bind: binding)
-            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
-          end
+        dirs = tree.keys.select {|k| tree[k].nil?} || []
+        Cnvrg::Logger.info("Sending Upload files save")
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+        unless Cnvrg::CLI.is_response_success(upload_resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
         end
         Cnvrg::Logger.log_info("Upload Success")
-        return files.try(:keys).try(:length), upload_error_files
+        return files.keys.length
       rescue => e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
       end
+
     end
 
     def upload_single_file(file)
       begin
-        file = file.as_json
-        Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
-        @downloader.safe_upload(file["path"], file["absolute_path"])
-        Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
+      file = file.as_json
+      Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
+      @downloader.upload(file["path"], file["absolute_path"])
+      Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
-        return false
       end
     end
 
@@ -830,7 +802,6 @@ module Cnvrg
       end
       return true
     end
-
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -838,6 +809,9 @@ module Cnvrg
       rescue => e
         return Cnvrg::Result.new(false, '')
       end
+
+      return Cnvrg::Result.new(true, '')
+
     end
 
     def download_dir(dataset_home, absolute_path)
@@ -882,7 +856,6 @@ module Cnvrg
       Cnvrg::CLI.is_response_success(response, true)
       return response
     end
-
     def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -890,24 +863,14 @@ module Cnvrg
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
-                                                                                username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
-        Cnvrg::CLI.is_response_success(response, true)
-        return response
-      rescue => e
-        return false
-      end
-    end
-    def last_valid_commit()
-      begin
-        #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
-        # create a new commit.
-        response = Cnvrg::API.request("#{base_resource}/last_valid_commit", 'GET')
-        Cnvrg::CLI.is_response_success(response, true)
-        return response
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
+                                                                              username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
+      Cnvrg::CLI.is_response_success(response, true)
+      return response
       rescue => e
         return false
       end
+
     end
 
     def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )

data/lib/cnvrg/dataset.rb

@@ -103,12 +103,8 @@ module Cnvrg
 
     def backup_idx
       Cnvrg::Logger.log_info("Backup idx")
-      if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        FileUtils.cp "#{self.local_path}/.cnvrg/idx.yml", "#{self.local_path}/.cnvrg/idx.yml.backup"
-      else
-        idx = {commit: nil, tree: {}}
-        File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
-      end
+      idx = self.get_idx
+      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
     end
 
     def restore_idx
@@ -784,6 +780,14 @@ module Cnvrg
       File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
     end
 
+    def get_current_commit()
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+        return nil
+      end
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      return idx_hash[:commit]
+    end
+
     def compare_commit(commit)
       if commit.nil? or commit.empty?
         commit = last_local_commit

data/lib/cnvrg/downloader/client.rb

@@ -2,8 +2,6 @@
 module Cnvrg
   module Downloader
     OLD_SERVER_VERSION_MESSAGE = "Your server version is not relevant for this cli version please contact support for further help."
-    MAXIMUM_BACKOFF = 64
-    RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 20
     attr_accessor :bucket, :client
     class Client
       def initialize(params)
@@ -43,35 +41,6 @@ module Cnvrg
         Cnvrg::Helpers.decrypt(@key, @iv, str)
       end
 
-      def safe_upload(storage_path, local_path)
-        n = 1
-        error = nil
-        while n <= RETRIES
-          begin
-            self.upload(storage_path, local_path)
-            error = nil
-            break
-          rescue => e
-            backoff_time_seconds = backoff_time(n)
-
-            message = "Got error: #{e.class.name} with message: #{e.message} while uploading a single file: #{local_path}, retry: #{n} of: #{RETRIES}"
-            if n < RETRIES
-              message += ", next retry in: #{backoff_time_seconds} seconds"
-            else
-              message += ", done retry, continuing to the next file"
-            end
-            Cnvrg::Logger.log_error_message(message)
-
-            sleep backoff_time_seconds
-
-            n += 1
-            error = e
-          end
-        end
-        raise error if error.present?
-        true
-      end
-
       def self.factory(params)
         params = params.as_json
         case params["storage"]
@@ -84,18 +53,6 @@ module Cnvrg
           return Cnvrg::Downloader::Clients::GcpClient.new(project_id: params["project_id"], credentials: params["credentials"], bucket_name: params["bucket_name"], sts: params["sts"])
         end
       end
-
-      private
-
-      def random_number_milliseconds
-        rand(1000) / 1000.0
-      end
-
-
-      def backoff_time(n)
-        return [((2**n)+random_number_milliseconds), MAXIMUM_BACKOFF].min
-      end
-
     end
   end
 end

data/lib/cnvrg/downloader/clients/azure_client.rb

@@ -20,11 +20,7 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          begin
-            client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
-          rescue => e
-            raise e
-          end
+          client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 10000)

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -38,11 +38,7 @@ module Cnvrg
         end
 
         def upload(storage_path, local_path)
-          begin
-            @bucket.create_file(local_path, storage_path)
-          rescue => e
-            raise e
-          end
+          @bucket.create_file(local_path, storage_path)
         end
       end
     end

data/lib/cnvrg/downloader/clients/s3_client.rb

@@ -53,10 +53,7 @@ module Cnvrg
           ### storage path is the path inside s3 (after the bucket)
           # local path is fullpath for the file /home/ubuntu/user.../hazilim.py
           o = aws_bucket.object(storage_path)
-          success = o.upload_file(local_path, @upload_options)
-          return success
-        rescue => e
-          raise e
+          o.upload_file(local_path, @upload_options)
         end
 
         def fetch_files(prefix: nil, marker: nil, limit: 1000)

data/lib/cnvrg/files.rb

@@ -6,7 +6,6 @@ require 'net/http'
 require 'cnvrg/result'
 module Cnvrg
   class Files
-    ParallelThreads = Cnvrg::Helpers.parallel_threads
     VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000
@@ -86,7 +85,6 @@ module Cnvrg
         Cnvrg::Logger.log_info("Upload files to older server..")
         return self.upload_files_old(files_list, commit_sha1, progress: progress)
       end
-
       files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
       resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
       unless Cnvrg::CLI.is_response_success(resp, false)
@@ -97,11 +95,10 @@ module Cnvrg
       files = res['files']
 
       #upload files
-      blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
+      blob_ids = Parallel.map(files.keys, self.get_upload_options) do |file|
         begin
           Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
         rescue => e
-
           Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
           Cnvrg::Logger.log_error(e)
           Cnvrg::Logger.log_method(bind: binding)
@@ -880,7 +877,6 @@ module Cnvrg
       end
 
     end
-
     def delete_commit_files_local(deleted)
       begin
         FileUtils.rm_rf(deleted) unless (deleted.nil? or deleted.empty?)
@@ -888,14 +884,16 @@ module Cnvrg
       rescue => e
         return false
       end
-    end
 
-    def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
-                                                                              username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
-      Cnvrg::CLI.is_response_success(response,false)
-      return response
+      return true
+
     end
+      def start_commit(new_branch,force:false, exp_start_commit:nil, job_slug: nil, job_type: nil, start_commit: nil, message: nil)
+        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,force:force,
+                                                                                username: @owner, exp_start_commit:exp_start_commit, job_slug: job_slug, job_type: job_type, start_commit: start_commit, message: message})
+        Cnvrg::CLI.is_response_success(response,false)
+        return response
+      end
 
     def end_commit(commit_sha1,force:false,message:"")
       response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force,message:message})

data/lib/cnvrg/project.rb

@@ -1,7 +1,4 @@
 require 'fileutils'
-require 'pathname'
-
-
 module Cnvrg
   class Project
     attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
@@ -329,13 +326,10 @@ module Cnvrg
     def get_storage_client
       response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
       if Cnvrg::CLI.is_response_success(response, false)
-
         client_params = response['client']
       else
-
         client_params = get_storage_client_fallback
       end
-
       Cnvrg::Downloader::Client.factory(client_params)
     end
 
@@ -433,8 +427,10 @@ module Cnvrg
       list_ignore_new = list_ignore.map{|x| x.gsub("//","/")} rescue []
       # list.each do |e|
       Parallel.map(list, in_threads: IDXParallelThreads) do |e|
-        label = e.sub(self.local_path + "/", "")
-
+        label = e.gsub(self.local_path + "/", "")
+        if not Cnvrg::Files.valid_file_name?(label)
+          raise StandardError.new("#{label} is not a valid file name")
+        end
         if list_ignore_new.include? label
           next
         end

data/lib/cnvrg/version.rb

@@ -1,3 +1,4 @@
 module Cnvrg
-  VERSION = '1.6.3.1'
+  VERSION = '1.6.10'
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.6.3.1
+  version: 1.6.10
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-01 00:00:00.000000000 Z
+date: 2020-01-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -210,14 +210,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.11.417
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.11.417
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: signet
   requirement: !ruby/object:Gem::Requirement