cnvrg 0.5.6 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 07964bc246a5f5139d9e726200e40d318a54eaec
-  data.tar.gz: f313dce679b50991e1440f668b8db27b1433bca8
+  metadata.gz: 5e3a5bae5049d633360023ea911ceeb97ed85696
+  data.tar.gz: 1f3b1959efd5784162d3d715b2b671188907297f
 SHA512:
-  metadata.gz: 56a184f578f4817a3f7471b3263d948575d7b685139748614793e1dad14f20419645279616c0bcffb43eaee31e7ce4d20ca7090ee080ade8167cfed668d80dca
-  data.tar.gz: d6dfd7f259df3aaaf55ccd4e93662e9d5b4f9ad184f138b14c605d09a3faed498d6d1f0dd3765ced81d966303bc5f696ba0bd262c8110809e3295d8a175e85f1
+  metadata.gz: 85643c72891b74c4282ce4991f8c7e572c1f21af287cb8883707ce1fbd34368a935e2508ac83610148f1caff3e86d1febd351dd0e129625da1a66e5ecf183414
+  data.tar.gz: ac285e4ddc22e20f4c7a811b59e82757e82f8686eaaae5d029a0aa2997bcb987283b1655a46dc7e87d1a44682cee4b504dcba99337064834ade2d80a60c8de04

data/lib/cnvrg/cli.rb

@@ -815,96 +815,49 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["result"]["name"]
         dataset_home = Dir.pwd+"/"+dataset_name
-
-
         if (Dir.exists? dataset_name)
           log_message("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
           if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
             log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
             exit(1)
           end
-
         end
         check = Helpers.checkmark
-
         if Dataset.clone(owner, dataset_name, slug)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           @files = Cnvrg::Datafiles.new(owner, slug)
-
-          successful_changes = []
           log_message("Downloading files", Thor::Shell::Color::BLUE)
-          if not(query.nil? or query.empty?)
-            @files.download_list_files_in_query(response["result"]["list_files"],dataset_home)
+          # if not(query.nil? or query.empty?)
+          #   @files.download_list_files_in_query(response["result"]["list_files"],dataset_home)
 
-            log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
-            return
+          #   log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+          #   return
 
-          end
+          # end
           commit = response["result"]["commit"]
-          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(dataset_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
-          if !response["result"]["tree"].nil?
-            parallel_options = {
-                :progress => {
-                    :title => "Download Progress",
-                    :progress_mark => '=',
-                    :format => "%b>>%i| %p%% %t",
-                    :starting_at => 0,
-                    :total => response["result"]["tree"].size,
-                    :autofinish => true
-                },
-                in_threads: ParallelThreads
-            }
-            begin
-              is_success = true
-
-              Parallel.map((response["result"]["tree"]), parallel_options) do |f|
-
-                relative_path = f[0].gsub(/^#{dataset_home}/, "")
-                if f[0].end_with? "/"
-                  # dir
-                  if @files.download_dir(dataset_home, relative_path)
-                    successful_changes << relative_path
-                  else
-                    is_success =false
-                    log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
-                    raise Parallel::Kill
-                  end
-                else
-                  # blob
-                  if !only_tree
-
-                  if @files.download_file_s3(f[0], relative_path, dataset_home, false, commit_sha1=commit)
-                    successful_changes << relative_path
-                  else
-                    is_success =false
-                    log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
-                    raise Parallel::Kill
-                  end
-                  end
-                end
-              end
-            rescue Interrupt
-              is_success =false
-              log_message("Couldn't download", Thor::Shell::Color::RED)
-
-              exit(1)
-            end
-
-
-          end
-          successful_changes = response["result"]["tree"]
-          if !successful_changes.nil? and is_success
+          files_count = response["result"]["file_count"]
+          files = @files.get_clone_chunk(commit: commit)
+          downloaded_files = 0
+          progressbar = ProgressBar.create(:title => "Download Progress",
+                                           :progress_mark => '=',
+                                           :format => "%b>>%i| %p%% %t",
+                                           :starting_at => 0,
+                                           :autofinish => true)
+          while files['keys'].length > 0
+            @files.download_multiple_files_s3(files, dataset_home)
+            downloaded_files += files['keys'].length
+            progressbar.progress = (downloaded_files * 100 / files_count)
+            files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
+          end
+          if downloaded_files == files_count
             Dataset.verify_cnvrgignore_exist(dataset_name, false)
-
+            @dataset = Dataset.new(dataset_home)
+            @dataset.write_idx(nil, commit)
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
           else
-            log_message("Couldn't download some files", Thor::Shell::Color::RED)
-
+            log_message(result.msg, result.msg_color)
           end
-
         else
-
           log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
           exit(1)
         end
@@ -1965,15 +1918,16 @@ module Cnvrg
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
     method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
+    method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
 
-    def sync_data_new(new_branch, force, verbose, commit, all_files, tags,parallel)
+    def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init)
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
-      if !options[:force]
+      if !force and !init
         invoke :download_data_new,[verbose,true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
-      invoke :upload_data_new,[new_branch,  verbose,true,force, tags], :new_branch=>new_branch,
+      invoke :upload_data_new,[new_branch,  verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
              :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
 
     end
@@ -1984,14 +1938,11 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
     method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
-
-    def upload_data_new(new_branch,  verbose,sync,force, tags)
+    def upload_data_new(new_branch,  verbose,sync,force, tags, chunk_size)
       commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force
-      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose,
-                          :force =>force, :sync =>sync,:parallel => options[:parallel]
-      if upload_res
-        invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force
-      end
+      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
+      invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force, :success => upload_res
+
       if tags
         log_message('Uploading Tags', Thor::Shell::Color::BLUE)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -2032,25 +1983,19 @@ module Cnvrg
         commit_sha1 = nil
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        next_commit = @dataset.get_next_commit()
-        if !next_commit.nil? and !next_commit.empty?
+        next_commit = @dataset.get_next_commit() #if there was a partial commit..
+        if next_commit.present? #keep from existing commit.
           resp = @files.get_commit(next_commit)
           if resp["result"]["status"].eql? "new"
             resp = @files.start_commit(new_branch,force)
             commit_sha1 = resp["result"]["commit_sha1"]
-
             @dataset.set_next_commit(commit_sha1)
-
           else
-
-              commit_sha1 = resp["result"]["commit_sha1"]
-
+            commit_sha1 = resp["result"]["commit_sha1"]
           end
         else
-
           resp = @files.start_commit(new_branch,force)
           commit_sha1 = resp["result"]["commit_sha1"]
-
           @dataset.set_next_commit(commit_sha1)
         end
         if direct
@@ -2059,16 +2004,14 @@ module Cnvrg
         return commit_sha1
       rescue => e
         puts e
-          if !commit_sha1.nil?
+          if commit_sha1.present?
             @dataset.set_next_commit(commit_sha1)
           end
       rescue SignalException
         log_message("Aborting..", Thor::Shell::Color::YELLOW)
-        if !commit_sha1.nil?
-
+        if commit_sha1.present?
           @dataset.set_next_commit(commit_sha1)
         end
-
         exit(1)
       end
 
@@ -2076,6 +2019,7 @@ module Cnvrg
     desc 'end_commit', 'start data commit'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :success, :type => :boolean,  :default => true, :hide=>true
 
     def end_commit_data(commit)
       begin
@@ -2085,12 +2029,18 @@ module Cnvrg
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         force = options["force"] || false
-
-        resp = @files.end_commit(commit, force)
+        resp = @files.end_commit(commit, force, options[:success])
         if (resp and resp["result"])
-          check = Helpers.checkmark()
-          @dataset.remove_next_commit()
-          log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+          if  !options[:success]
+            @dataset.set_next_commit(resp["result"]["commit_sha1"])
+            @dataset.remove_next_commit
+          else
+            check = Helpers.checkmark
+            @dataset.set_next_commit(commit)
+            @dataset.remove_next_commit
+            log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+          end
+
           end
 
 
@@ -2144,6 +2094,7 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :tags, :type => :string, :aliases => ["--tags"], :desc => "upload file tags", :default => ""
+    method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :desc => "upload file tags", :default => 1000
     # method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
@@ -2153,219 +2104,242 @@ module Cnvrg
         verify_logged_in(true)
         log_start(__method__, args, options)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
-
         @dataset = Dataset.new(dataset_dir)
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         next_commit = @dataset.get_next_commit()
-
-
-        if (new_commit.nil? or new_commit.empty?) and next_commit.nil?
+        if new_commit.blank? and next_commit.blank?
           log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
           exit(1)
         end
 
         force = options[:force] || false
-        parallel_threads = options["parallel"]  || ParallelThreads
+        chunk_size = options[:chunk_size]
+        chunk_size = [chunk_size, 1].max
         new_branch = options["new_branch"] || false
-          log_message("Checking dataset", Thor::Shell::Color::BLUE) unless options[:sync]
-          local_idx = @dataset.generate_idx(true)
-          result = @dataset.compare_idx(new_branch, commit=@dataset.last_local_commit, local_idx= local_idx, force=force, next_commit= next_commit)
-
-        commit_sha1 = result["result"]["commit"]
-          if commit_sha1 != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-            @dataset.remove_next_commit()
-            @files.delete_commit(next_commit)
-            log_message("Remote server has an updated version, please run `cnvrg data download` first or cnvrg data upload --new_branch", Thor::Shell::Color::BLUE)
-            exit(1)
-          end
-        check = Helpers.checkmark()
+        force = options["force"] || false
+        parallel_threads = options["parallel"]  || ParallelThreads
+        new_tree = {}
+        progressbar = ProgressBar.create(:title => "Upload Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :autofinish => true)
+        all_files = @dataset.list_all_files
+        files_count = all_files.count
+        finished_chunks = 0
+        files_uploaded = 0
+        all_files.each_slice(chunk_size).each do |list_files|
+          temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+          files_uploaded += @files.upload_multiple_files(next_commit, temp_tree, threads: parallel_threads, force: force)
+          new_tree = new_tree.merge(temp_tree.map{|k,v| [k.gsub('./', ''), {sha1: v[:sha1], commit_time: nil}]}.to_h)
+          @dataset.write_idx(new_tree, next_commit)
+          finished_chunks += 1
+          progressbar.progress = [(finished_chunks * chunk_size*100) / files_count, 100].min
+        end
+        return files_uploaded > 0
+      rescue Exception => e
+        return false
+      end
 
+      #   local_idx = @dataset.generate_idx
+      #   result = @dataset.compare_idx(new_branch, commit=@dataset.last_local_commit, local_idx= local_idx, force=force, next_commit= next_commit)
 
-        commit_time = result["result"]["commit_time"]
-        already_uploaded = result["result"]["uploaded"]
-        dirs_uploaded = []
-        files_uploaded = []
-        files_uploaded_names = []
-        if !already_uploaded.nil? and !already_uploaded.empty?
+      #   commit_sha1 = result["result"]["commit"]
+      #     if commit_sha1 != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+      #       @dataset.remove_next_commit()
+      #       @files.delete_commit(next_commit)
+      #       log_message("Remote server has an updated version, please run `cnvrg data download` first or cnvrg data upload --new_branch", Thor::Shell::Color::BLUE)
+      #       exit(1)
+      #     end
+      #   check = Helpers.checkmark()
 
-          dirs_uploaded = already_uploaded["dirs"]
-          files_uploaded = already_uploaded["blobs"]
-          if !files_uploaded.nil? and !files_uploaded.empty?
-            files_uploaded_names = already_uploaded["blobs"].keys
-          end
 
-        end
+      #   commit_time = result["result"]["commit_time"]
+      #   already_uploaded = result["result"]["uploaded"]
+      #   dirs_uploaded = []
+      #   files_uploaded = []
+      #   files_uploaded_names = []
+      #   if !already_uploaded.nil? and !already_uploaded.empty?
 
-        result = result["result"]["tree"]
-        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          @files.delete_commit(next_commit)
+      #     dirs_uploaded = already_uploaded["dirs"]
+      #     files_uploaded = already_uploaded["blobs"]
+      #     if !files_uploaded.nil? and !files_uploaded.empty?
+      #       files_uploaded_names = already_uploaded["blobs"].keys
+      #     end
 
-          log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, true)
-          return false
-        end
-        log_message("Creating new commit for dataset: #{@dataset.slug}", Thor::Shell::Color::BLUE) unless  options[:sync]
+      #   end
 
+      #   result = result["result"]["tree"]
+      #   if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+      #     @files.delete_commit(next_commit)
 
-        # upload / update
-        begin
-          parallel_options = {
-              :progress => {
-                  :title => "Upload Progress",
-                  :progress_mark => '=',
-                  :format => "%b>>%i| %p%% %t",
-                  :starting_at => 0,
-                  :total => (result["added"] + result["updated_on_local"]).size,
-                  :autofinish => true
-              },
-              in_threads: parallel_threads.to_i,
-              isolation: true
-          }
-          successful_updates = []
-          update_count =0
-          update_total = (result["added"] + result["updated_on_local"] + result["deleted"]).size
+      #     log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, true)
+      #     return false
+      #   end
+      #   log_message("Creating new commit for dataset: #{@dataset.slug}", Thor::Shell::Color::BLUE) unless  options[:sync]
 
-          if (result["added"] + result["updated_on_local"]).size > 0
-            begin
-              upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
-                absolute_path = "#{@dataset.local_path}/#{f}"
-                relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-                if File.directory?(absolute_path)
-                  if dirs_uploaded.include? f
-                    #Already uploaded and didn't finish commit
-                    log_message("Already uploaded dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
 
-                    next
-                  end
-                  log_message("uploading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+      #   # upload / update
+      #   begin
 
-                  resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-                  if resDir
-                    update_count += 1
-                    successful_updates<< relative_path
-                  else
-                    log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
+      #     parallel_options = {
+      #         :progress => {
+      #             :title => "Upload Progress",
+      #             :progress_mark => '=',
+      #             :format => "%b>>%i| %p%% %t",
+      #             :starting_at => 0,
+      #             :total => (result["added"] + result["updated_on_local"]).size,
+      #             :autofinish => true
+      #         },
+      #         in_threads: ParallelThreads,
+      #         isolation: true
+      #     }
+      #     successful_updates = []
+      #     update_count =0
+      #     update_total = (result["added"] + result["updated_on_local"] + result["deleted"]).size
 
-                    raise Parallel::Kill
-                  end
+      #     if (result["added"] + result["updated_on_local"]).size > 0
+      #       begin
+      #         upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
+      #           absolute_path = "#{@dataset.local_path}/#{f}"
+      #           relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+      #           if File.directory?(absolute_path)
+      #             if dirs_uploaded.include? f
+      #               #Already uploaded and didn't finish commit
+      #               log_message("Already uploaded dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
 
-                else
-                  if files_uploaded_names.include? f
-                    log_message("Already uploaded file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+      #               next
+      #             end
+      #             log_message("uploading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
 
-                    next
-                  end
-                  log_message("uploading: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+      #             resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+      #             if resDir
+      #               update_count += 1
+      #               successful_updates<< relative_path
+      #             else
+      #               log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
 
-                  res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-                  if res
-                    update_count += 1
+      #               raise Parallel::Kill
+      #             end
 
-                    successful_updates<< relative_path
-                  else
-                    log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
+      #           else
+      #             if files_uploaded_names.include? f
+      #               log_message("Already uploaded file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
 
-                    raise Parallel::Kill
+      #               next
+      #             end
+      #             log_message("uploading: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
 
-                  end
-                end
-              end
-            rescue SignalException
-              log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+      #             res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+      #             if res
+      #               update_count += 1
 
+      #               successful_updates<< relative_path
+      #             else
+      #               log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
 
-              exit(1)
-            end
-          end
+      #               raise Parallel::Kill
 
-        end
-          successful_updates = upload_result.to_a + dirs_uploaded + files_uploaded_names
-        successful_deletions =[]
+      #             end
+      #           end
+      #         end
+      #       rescue SignalException
+      #         log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
 
 
-          # delete
+      #         exit(1)
+      #       end
+      #     end
 
-          deleted = update_deleted(result["deleted"])
-          begin
+      #   end
+      #     successful_updates = upload_result.to_a + dirs_uploaded + files_uploaded_names
+      #   successful_deletions =[]
 
-            deleted_result = Parallel.map(deleted, in_threads: ParallelThreads) do |f|
 
-              relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-              if relative_path.end_with?("/")
-                log_message("deleting dir: #{f}", Thor::Shell::Color::RED, options["verbose"])
+      #     # delete
 
-                if @files.delete_dir(f, relative_path, commit_sha1)
-                  successful_deletions << f
-                else
-                  log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
+      #     deleted = update_deleted(result["deleted"])
+      #     begin
 
-                end
-              else
-                log_message("deleteing file: #{f}", Thor::Shell::Color::RED, options["verbose"])
+      #       deleted_result = Parallel.map(deleted, in_threads: ParallelThreads) do |f|
 
-                if @files.delete_file(f, relative_path, commit_sha1)
-                  successful_deletions << f
+      #         relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+      #         if relative_path.end_with?("/")
+      #           log_message("deleting dir: #{f}", Thor::Shell::Color::RED, options["verbose"])
 
-                else
-                  log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
+      #           if @files.delete_dir(f, relative_path, commit_sha1)
+      #             successful_deletions << f
+      #           else
+      #             log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
+      #             raise Parallel::Kill
 
+      #           end
+      #         else
+      #           log_message("deleteing file: #{f}", Thor::Shell::Color::RED, options["verbose"])
 
-                end
-              end
-            end
+      #           if @files.delete_file(f, relative_path, commit_sha1)
+      #             successful_deletions << f
 
+      #           else
+      #             log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
+      #             raise Parallel::Kill
 
-          successful_deletions += successful_deletions.select { |x| not x.nil? }
 
+      #           end
+      #         end
+      #       end
 
-          successful_updates = successful_updates.select { |x| not x.nil? }
 
-          update_count = successful_updates.size
+      #     successful_deletions += successful_deletions.select { |x| not x.nil? }
 
-        rescue SignalException
-          say "User aborted", Thor::Shell::Color::RED
-          exit(0)
-        rescue => e
-          log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-          log_error(e)
-          exit(0)
-        end
-        if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count += result["deleted"].size
-        end
 
-        if update_count == update_total
-          begin
-            @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
+      #     successful_updates = successful_updates.select { |x| not x.nil? }
 
-            return true
-          rescue => e
-            log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+      #     update_count = successful_updates.size
 
-            exit(1)
+      #   rescue SignalException
+      #     say "User aborted", Thor::Shell::Color::RED
+      #     exit(0)
+      #   rescue => e
+      #     log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+      #     log_error(e)
+      #     exit(0)
+      #   end
+      #   if !result["deleted"].nil? and !result["deleted"].empty?
+      #     update_count += result["deleted"].size
+      #   end
 
-          end
+      #   if update_count == update_total
+      #     begin
+      #       @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
 
-        end
-      rescue => e
+      #       return true
+      #     rescue => e
+      #       log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
 
-        log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-        log_error(e)
+      #       exit(1)
 
-        exit(1)
-      rescue SignalException
+      #     end
 
-        say "\nAborting", Thor::Shell::Color::BLUE
-        exit(1)
-      end
+      #   end
+      # rescue => e
+
+      #   log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+      #   log_error(e)
+
+      #   exit(1)
+      # rescue SignalException
+
+      #   say "\nAborting", Thor::Shell::Color::BLUE
+      #   exit(1)
+      # end
 
     end
 
 
     desc 'upload', 'Upload updated files'
     method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--new_branch"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -2374,12 +2348,14 @@ module Cnvrg
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
+    method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false)
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false,output_dir="output",job_type=nil,job_slug=nil )
       begin
+        # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
         log_start(__method__, args, options)
-
         @project = Project.new(get_project_home)
         commit_msg = options["message"]
         if commit_msg.nil? or commit_msg.empty?
@@ -2388,18 +2364,24 @@ module Cnvrg
         return_id = options["return_id"]
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
         ignore = options[:ignore] || ""
-        force = options[:force] || false
+        force = options[:force] || force || false
         spec_files_to_upload = options["files"]
+        check = Helpers.checkmark()
+
         if !spec_files_to_upload.blank?
           spec_files_to_upload = spec_files_to_upload.split(",")
         end
-        git_output_dir = options["output_dir"]
-        if !git_output_dir.blank?
+        git_output_dir = options["output_dir"] || output_dir
+        if git_output_dir.present?
           if git_output_dir.ends_with? "/"
             git_output_dir = git_output_dir[0..-2]
           end
           spec_files_to_upload = Dir.glob("#{git_output_dir}/**/*", File::FNM_DOTMATCH).flatten
           spec_files_to_upload = spec_files_to_upload.select{|x| !x.ends_with? "/."}
+          if spec_files_to_upload.blank?
+            log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+              return true
+          end
           force = true
         end
 
@@ -2431,7 +2413,7 @@ module Cnvrg
         end
         result = result["result"]["tree"]
 
-        check = Helpers.checkmark()
+
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
           return true
@@ -2448,7 +2430,6 @@ module Cnvrg
           end
         else
           log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
-
         end
         # Start commit
         if in_exp
@@ -2456,8 +2437,10 @@ module Cnvrg
         else
           exp_start_commit = nil
         end
-        commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit:exp_start_commit)["result"]["commit_sha1"]
-
+        job_type = options['job_type'] || job_type
+        job_slug = options['job_slug'] || job_slug
+        commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit:exp_start_commit,
+                                          job_type: job_type, job_slug: job_slug)["result"]["commit_sha1"]
         # upload / update
         begin
 
@@ -2753,8 +2736,8 @@ module Cnvrg
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => ""
     method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
 
+    def download_data_new(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
       begin
-        def download_data_new(verbose=false,sync=false, commit=nil,all_files=true)
         verify_logged_in(true)
         log_start(__method__, args, options)
         dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -2763,23 +2746,14 @@ module Cnvrg
         commit_to_download = commit
         local_idx = @dataset.generate_idx
         all_files = all_files
-        if !commit_to_download.nil? and !commit_to_download.empty?
-          res = @dataset.compare_idx_download( commit=commit_to_download, local_idx= local_idx,all_files=all_files)
-        else
-          res = @dataset.compare_idx_download( commit=nil, local_idx= local_idx,all_files=all_files)
-
-        end
-
+        res = @dataset.compare_idx_download(commit=commit_to_download.presence, local_idx= local_idx,all_files=all_files)
         result = res["result"]["tree"]
-
         commit = res["result"]["commit"]
         if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
           log_message("Dataset is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
           return true
         end
         update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
-
-
         successful_changes = 0
         if update_total ==1
           log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !sync)
@@ -2790,112 +2764,20 @@ module Cnvrg
           log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
         else
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
-
-        end
-        parallel_options = {
-            :progress => {
-                :title => "Download Progress",
-                :progress_mark => '=',
-                :format => "%b>>%i| %p%% %t",
-                :starting_at => 0,
-                :total => result["updated_on_server"].size,
-                :autofinish => true
-            },
-            in_threads: ParallelThreads
-        }
-        conflicts = 0
-        if !result["conflicts"].empty?
-          begin
-
-
-            conflicts_result = Parallel.map(result["conflicts"], in_threads: ParallelThreads) do |f|
-
-              relative_path = f.gsub(/^#{@dataset.local_path}/, "")
-              log_message("downloading: #{f}.conflict", Thor::Shell::Color::BLUE, verbose)
-
-              if @files.download_file_s3(f, relative_path, dataset_dir,  conflict=true, commit_sha1=commit)
-                conflicts +=1
-              else
-                log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
-                raise Parallel::Kill
-
-              end
-            end
-          rescue Interrupt
-
-            log_message("Failed to download files", Thor::Shell::Color::RED)
-
-            exit(1)
-          end
         end
-
-
-        successful_changes += conflicts
-        downloads = 0
-        if !result["updated_on_server"].empty?
-
-            updated_on_server_result = Parallel.map(result["updated_on_server"], parallel_options) do |f|
-              begin
-              relative_path = f.gsub(/^#{@dataset.local_path}/, "")
-              if f.end_with? "/"
-                # dir
-                if !all_files
-                  log_message("downloading dir: #{f}", Thor::Shell::Color::BLUE, verbose)
-                  if @files.download_dir(dataset_dir, relative_path)
-                    downloads +=1
-                  else
-                    log_message("Couldn't create directory: #{f}", Thor::Shell::Color::RED)
-                    raise Parallel::Kill
-
-
-                  end
-                end
-
-              else
-                # blob
-                log_message("downloading file: #{f}", Thor::Shell::Color::BLUE, verbose)
-
-                if @files.download_file_s3(f, relative_path, dataset_dir,conflict=false, commit_sha1=commit)
-                  downloads +=1
-
-                else
-
-                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
-
-
-                end
-              end
-              rescue Interrupt
-                log_message("Couldn't download", Thor::Shell::Color::RED)
-
-                exit(1)
-
-              end
-
-            end
-            successful_changes += downloads
-
-        end
-
+        update_res = @files.download_files_in_chunks(result["updated_on_server"]) if result["updated_on_server"].present?
+        conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
         deleted = result["deleted"].to_a
         delete_res = @files.delete_commit_files_local(deleted)
         if !delete_res
           log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
           log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
-
           exit(1)
-
         end
-        successful_changes += deleted.size
-
-
-
-        if update_total == successful_changes
+        if [update_res.is_success?, delete_res.is_success?, (conflict_res.blank? or conflict_res.is_success?)].all?
           # update idx with latest commit
           @dataset.update_idx_with_commit!(commit)
           check = Helpers.checkmark()
-
           if options["verbose"]
             log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
             log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
@@ -2904,10 +2786,8 @@ module Cnvrg
             log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
           return true
-
         end
       rescue => e
-
         log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         log_error(e)
         exit(1)
@@ -2915,7 +2795,8 @@ module Cnvrg
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       end
-      end
+    end
+
     desc 'download in git project', 'Download other files', :hide =>true
     def download_in_git(commit_sha1)
       begin
@@ -3322,27 +3203,33 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["-m", "--message"], :default => ""
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
-    method_option :in_exp, :type => :boolean, :aliases => ["-e", "--in_exp"], :default => false
+    method_option :in_exp, :type => :boolean, :aliases => ["-e", "--in_exp"], :default => false #deprecated..
+    method_option :job_slug, :type => :string, :aliases => ["-j", "--job"], :default => nil
+    method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
-
-
     def sync(direct = true)
       verify_logged_in(true) if direct
       log_start(__method__, args, options)
       log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
       log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
-      if !options[:force] and (options[:files].blank? or options[:output_dir].blank?)
-        invoke :download, [true, "", options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
+      job_slug = options['job_slug']
+      job_type = options['job_type']
+      in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
+      run_download = true
+      if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
+        run_download =false
+      end
+      if run_download
+        invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
-      invoke :upload, [false, true,  direct, "",options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+      invoke :upload, [false, true,  direct, "",options['exp_slug'] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
-             :files => options["files"], :output_dir => options["output_dir"]
+             :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
 
 
     end
 
-
     desc 'run cmd', 'Runs an experiment'
     method_option :local, :type => :boolean, :aliases => ["-l", "--local"], :default => false
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
@@ -3487,10 +3374,8 @@ module Cnvrg
       force = options[:force]
       sync_before_terminate = options["sync_before_terminate"]
       periodic_sync = options["periodic_sync"]
-
-
-
       email_notification = options["email_notification"]
+      output_dir = options['output_dir'] || "output"
       project_home = get_project_home
       data_query = options["data_query"]
       @project = Project.new(project_home)
@@ -3506,6 +3391,7 @@ module Cnvrg
           if sync_before
             # Sync before run
             invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore, :force => force
+
           end
         end
         #set image for the project
@@ -3578,10 +3464,9 @@ module Cnvrg
                             if output_dir.blank?
                               output_dir = "output"
                             end
-                            invoke :upload, [false, true, false, ignore, true, true], :output_dir => output_dir, :force=>true
+                            upload(false, false, true, ignore, true, true,output_dir,"Experiment",@exp.slug  )
                           else
-                            upload_res = upload(false, true,  false,  ignore, true, true)
-
+                            upload(false, false, true, ignore, true, true,nil,"Experiment",@exp.slug  )
                           end
                             res = @exp.send_restart_request(@project.get_idx.try(:fetch, :commit))
                           while !Cnvrg::CLI.is_response_success(res, false) do
@@ -3685,24 +3570,17 @@ module Cnvrg
 
                 if sync_after
                 # Sync after run
-                upload_res_count = 0
-                upload_res = false
-                  while !upload_res and upload_res_count <5
-                    if @project.is_git
-                      if !upload_output.blank?
-                        upload_output = upload_output.join(",")
-                        invoke :upload, [false, true, false, ignore, true, true], :output_dir => upload_output, :force=>true
-                      end
-
-                    else
-                      upload_res = upload(false, true,  false,  ignore, true, false)
-
-
+                  if @project.is_git
+                    output_dir = output_dir || @exp.output_dir
+                    if output_dir.present?
+                      upload(false, false, true, ignore, true, true,output_dir,"Experiment",@exp.slug  )
+                      # invoke :upload, [false, false, true, ignore, true, true], :output_dir => output_dir, :force=>true, :job_type=>'Experiment', :job_slug=>@exp.slug
                     end
-                    upload_res_count +=1
+                  else
+                    upload(false, false, true, ignore, true, true,nil,"Experiment",@exp.slug  )
+
+                    # invoke :upload, [false, false, true, ignore,true, true], :job_type=>'Experiment', :job_slug=>@exp.slug, :force=>true
                   end
-                # download(sync = true, ignore_list = ignore)
-                # upload(link = false, sync = true, direct = false, ignore_list = ignore)
 
                 end
                 end_commit = @project.last_local_commit
@@ -3710,7 +3588,7 @@ module Cnvrg
                 # log_thread.join
                 stats_thread.join
 
-              res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
 
 
                 if !exp_success
@@ -3865,6 +3743,7 @@ module Cnvrg
 
         end
         project = Project.new(working_dir)
+
         if project.is_git and output_dir.blank?
           output_dir  = "output"
         end