cnvrg 0.7.7 → 0.7.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dd18cf5dd5400ab69be84219df510ab65229aac4
-  data.tar.gz: f06a47bf63d07b71b432af21e268c9b6bc7f57ab
+  metadata.gz: 2b19ed887eb07e3f10c710d73849c7578cf10824
+  data.tar.gz: 397910ae8ce973da74c4b921ab6fc2e59ea63388
 SHA512:
-  metadata.gz: 9447ae7930c2291b33801f474942e72c0eb90e2e39b37ebd3c5fbb7f040bab29942e044aa21aef58afcc8624569b8bd40be57cdb5739eaf897aeaf5b50cb7be6
-  data.tar.gz: 5d0b5e162a53b62ea170e0bd5d2c53c15ef10a2c00bc71d1a4b5a7237494d6012c15b86b372825f3a294b12e3e7c6138db2121eb6ce5d8a825b190032b80775d
+  metadata.gz: 47f085ae30d741751dfca95a8196cc794460ce36cb1cb9e2847a577cb94936488f8c2abd0571d3cc6e4a894aa01bf2d8fb1efbb7fe8199770bdaca92466ed91a
+  data.tar.gz: '06699e08c81ed1626ef49ac09f2a5b8deb90b29c41ecd3b1d6b6984c5f4f8351848aea342b397b3186b7dbfffab0459bccbfa9c6fbf744a94c16810d2914a449'

data/cnvrg.gemspec

@@ -45,10 +45,4 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'net-ssh'
     spec.add_runtime_dependency 'down'
-
-
-
-
-    
-end
-
+end

data/lib/cnvrg/api.rb

@@ -61,8 +61,9 @@ module Cnvrg
                 success = false
                 while !success and retries < 20
                   begin
-                response = conn.get "#{resource}", data
+                    response = conn.get "#{resource}", data
                     success = true
+                    Cnvrg::API.parse_version(response)
                   rescue => e
                     sleep(5)
                     retries +=1
@@ -89,6 +90,7 @@ module Cnvrg
                         response = conn.post "#{resource}", data if method.eql? 'POST'
                         response = conn.put "#{resource}", data if method.eql? 'PUT'
                         success = true
+                        Cnvrg::API.parse_version(response)
 
                     rescue => e
                         sleep(5)
@@ -119,6 +121,7 @@ module Cnvrg
                 begin
                   response = conn.post "#{resource}", new_data
                   success = true
+                  Cnvrg::API.parse_version(response)
 
                 rescue => e
                   sleep(5)
@@ -171,6 +174,7 @@ module Cnvrg
                 data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
 
                 response = conn.post "#{endpoint_uri}/#{resource}", data
+                Cnvrg::API.parse_version(response)
 
                 FileUtils.rm_rf(temp_path)
                 if response.to_hash[:status] == 404
@@ -185,6 +189,7 @@ module Cnvrg
                 end
             when 'DELETE'
                 response = conn.delete "#{endpoint_uri}/#{resource}", data
+                Cnvrg::API.parse_version(response)
                 if response.to_hash[:status] == 404
                   return false
                 end
@@ -215,6 +220,15 @@ module Cnvrg
             "Oops, an error occurred! Reason: #{response['message']}"
         end
 
+      def self.parse_version(resp)
+        begin
+          version = resp.headers["cnvrg-version"]
+          Cnvrg::Helpers.update_version(version)
+        rescue => e
+          Cnvrg::Logger.log_error(e)
+        end
+      end
+
         # Internal: Ensure the response returns a HTTP 200.
         #
         # If the response status isn't a HTTP 200, we need to find out why. This

data/lib/cnvrg/cli.rb

@@ -23,6 +23,9 @@ require 'cnvrg/datafiles'
 require 'cnvrg/data'
 require 'cnvrg/ssh'
 require 'cnvrg/result'
+require 'cnvrg/logger'
+require 'cnvrg/cli/flow'
+require 'cnvrg/cli/task'
 require 'logstash-logger'
 require 'cnvrg/job'
 require 'docker'
@@ -157,9 +160,18 @@ module Cnvrg
         super
       end
     end
-    desc "data", "upload and manage datasets", :hide => true
+    desc "data", "upload and manage datasets", :hide => false
     subcommand "data", Data
 
+    # desc "flow", "mange project flows", :hide => true
+    # subcommand "flow", Cnvrg::Commands::Flow
+
+
+    def initialize(*args)
+      super
+      self.log_handler
+    end
+
     desc "", "", :hide => true
 
     def test
@@ -207,7 +219,7 @@ module Cnvrg
       owner = url_parts[project_index - 1]
       res = Cnvrg::API.request("users/#{owner}/datasets/#{slug}", 'GET')
       unless Cnvrg::CLI.is_response_success(res, false)
-        raise SignalException.new()
+        raise SignalException.new
       end
       @dataset = Dataset.new(Dir.pwd)
       result = res['result']
@@ -873,7 +885,7 @@ module Cnvrg
                                            :autofinish => true)
 
           while files['keys'].length > 0
-
+            Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
             @files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar, read_only: read)
 
             downloaded_files += files['keys'].length
@@ -1249,7 +1261,7 @@ module Cnvrg
         #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
         #   exit(1)
         #
-        # end
+        # en
         check = Helpers.checkmark()
 
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
@@ -1804,9 +1816,7 @@ module Cnvrg
           @files = Cnvrg::Files.new(@project.owner, slug)
           response = @project.clone(remote, commit_to_clone)
           Cnvrg::CLI.is_response_success response
-          working_dir = project_home
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(working_dir + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
 
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           files = response["result"]["files"]
@@ -1814,6 +1824,7 @@ module Cnvrg
           if !files.nil?
             begin
               download_result = @files.download_multpile_files_s3(files, project_home)
+              @project.set_idx(idx)
 
             rescue Interrupt
               log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
@@ -1876,6 +1887,7 @@ module Cnvrg
               end
               successful_changes = response["result"]["tree"]
               if !successful_changes.nil? and is_success
+                @project.set_idx(idx)
                 Project.verify_cnvrgignore_exist(project_name,remote)
                 log_message("Done.\nDownloaded  #{successful_changes.size}/#{response["result"]["tree"].size} files", Thor::Shell::Color::GREEN)
               else
@@ -1919,6 +1931,7 @@ module Cnvrg
         verify_logged_in()
         log_start(__method__, args, options)
         @project = Project.new(get_project_home)
+
         new_branch = options["new_branch"] || false
         force = options["force"] || false
 
@@ -2030,9 +2043,6 @@ module Cnvrg
       if upload_res < 0
         return
       end
-      # if !upload_res
-      #   return
-      # end
       invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res] , :new_branch=>new_branch,  :force =>force
       if tags
         log_message('Uploading Tags', Thor::Shell::Color::BLUE)
@@ -2051,70 +2061,57 @@ module Cnvrg
           log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
         end
       end
-    rescue => e
-      say "\nAborting", Thor::Shell::Color::BLUE
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      return false if dataset_dir.blank?
-      @dataset = Dataset.new(dataset_dir)
-      @dataset.revert_to_last_commit
-      return false
-    rescue SignalException
-      say "\nAborting", Thor::Shell::Color::BLUE
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      return false if dataset_dir.blank?
-      @dataset = Dataset.new(dataset_dir)
-      @dataset.revert_to_last_commit
-      return false
-    end
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        say "\nAborting", Thor::Shell::Color::BLUE
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        return false if dataset_dir.blank?
+        @dataset = Dataset.new(dataset_dir)
+        return false
+      rescue SignalException => e
+        Cnvrg::Logger.log_error(e)
+        say "\nAborting", Thor::Shell::Color::BLUE
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        return false if dataset_dir.blank?
+        @dataset = Dataset.new(dataset_dir)
+        return false
+      end
     end
 
     desc 'start_commit', 'start data commit'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
-    method_option :direct, :type => :boolean, :aliases => ["-d","--direct"], :desc => "was called directed", :default => true
-    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :direct, :type => :boolean, :aliases => ["-d", "--direct"], :desc => "was called directed", :default => true
+    method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
     method_option :chunk_size, :type => :numeric, :aliases => ["-ch"], :default => 0
 
     def start_commit_data()
-      begin
-        verify_logged_in(true)
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        direct = options[:direct]
-        new_branch = options["new_branch"] || false
-        force = options["force"] || false
-        chunk_size = options["chunk_size"] || false
-        commit_sha1 = nil
-        @dataset = Dataset.new(dataset_dir)
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        next_commit = @dataset.get_next_commit #if there was a partial commit..
-        chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
-        resp = @files.start_commit(new_branch, force, chunks: chunks, current_commit: next_commit || @dataset.last_local_commit)
-        if !resp['result']['can_commit']
-          log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
-          exit(1)
-        end
-        #  if next_commit.present? #keep from existing commit.
-        #           resp = @files.get_commit(next_commit)
-        #           if resp["result"]["status"].eql? "new"
-        #             resp = @files.start_commit(new_branch,force, chunks: chunks)
-        #             commit_sha1 = resp["result"]["commit_sha1"]
-        #             @dataset.set_next_commit(commit_sha1)
-        #           else
-        #             commit_sha1 = resp["result"]["commit_sha1"]
-        #           end
-        #         else
-        #           resp = @files.start_commit(new_branch,force, chunks: chunks)
-        #           commit_sha1 = resp["result"]["commit_sha1"]
-        #           @dataset.set_next_commit(commit_sha1)
-        #         end
-        commit_sha1 = resp["result"]["commit_sha1"]
-        @dataset.set_next_commit(commit_sha1)
-        if direct
-          puts commit_sha1
-        end
-        return commit_sha1
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      direct = options[:direct]
+      new_branch = options["new_branch"] || false
+      force = options["force"] || false
+      chunk_size = options["chunk_size"] || false
+      commit_sha1 = nil
+      @dataset = Dataset.new(dataset_dir)
+      @dataset.backup_idx
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+      next_commit = @dataset.get_next_commit #if there was a partial commit..
+      chunks = (@dataset.list_all_files.length.to_f / chunk_size).ceil
+      resp = @files.start_commit(new_branch, force, chunks: chunks, dataset: @dataset)
+      if !resp['result']['can_commit']
+        log_message("Cant upload files because a new version of this dataset exists, please download it or upload with --force", Thor::Shell::Color::RED)
+        exit(1)
+      end
+      commit_sha1 = resp["result"]["commit_sha1"]
+      unless commit_sha1.eql? next_commit
+        @dataset.set_partial_commit(next_commit)
       end
+      @dataset.set_next_commit(commit_sha1)
+      return commit_sha1
     end
+
+
     desc 'end_commit', 'start data commit'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
@@ -2128,15 +2125,16 @@ module Cnvrg
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         force = options["force"] || false
         resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
-        if (resp and resp["result"])
+        if (resp.present? and resp["result"])
           check = Helpers.checkmark
           if resp["result"]["new_commit"].blank?
-            @dataset.revert_next_commit
+            @dataset.revert_next_commit #removes the next commit
             log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
           else
             log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
-            @dataset.set_next_commit(resp["result"]["commit_sha1"]) # no need fo rboth methods FIX
-            @dataset.remove_next_commit()
+            @dataset.remove_next_commit #takes the next commit and put it as current commit
+            @dataset.set_partial_commit(nil)
+            @dataset.backup_idx
           end
         end
       end
@@ -2188,253 +2186,59 @@ module Cnvrg
 
     def upload_data_files(new_commit, *files)
       begin
-        verify_logged_in(true)
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        new_commit ||= @dataset.get_next_commit()
-        if new_commit.blank?
-          log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
-          return false
-        end
-        force = options[:force] || false
-        chunk_size = options[:chunk_size]
-        chunk_size = [chunk_size, 1].max
-        new_branch = options["new_branch"] || false
-        new_tree = {}
-        force = options["force"] || false
-        parallel_threads = options["parallel"]  || ParallelThreads
-        all_files = @dataset.list_all_files
-        progressbar = ProgressBar.create(:title => "Upload Progress",
-                                         :progress_mark => '=',
-                                         :format => "%b>>%i| %p%% %t",
-                                         :starting_at => 0,
-                                         :total => all_files.length,
-                                         :autofinish => true)
-        files_uploaded = 0
-        all_files.each_slice(chunk_size).each do |list_files|
-          temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
-
-          upload_resp = @files.upload_multiple_files(new_commit, temp_tree, threads: parallel_threads, force: force, new_branch: new_branch, progressbar: progressbar)
-
-          if upload_resp.is_a? Cnvrg::Result
-            log_message(upload_resp.msg, upload_resp.msg_color)
-            raise SignalException.new(upload_resp.msg)
-            # return -1
-
-          else
-            files_uploaded += upload_resp
-            next if upload_resp == 0
-          end
-
-          temp_tree.each do |k,v|
-            new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
-          end
-          @dataset.write_idx(new_tree, new_commit)
-        end
-        return files_uploaded
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+      new_commit ||= @dataset.get_next_commit()
+      partial_commit = @dataset.get_partial_commit
+      if new_commit.blank?
+        log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
+        return false
       end
-
-      #   local_idx = @dataset.generate_idx
-      #   result = @dataset.compare_idx(new_branch, commit=@dataset.last_local_commit, local_idx= local_idx, force=force, next_commit= next_commit)
-
-      #   commit_sha1 = result["result"]["commit"]
-      #     if commit_sha1 != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-      #       @dataset.remove_next_commit()
-      #       @files.delete_commit(next_commit)
-      #       log_message("Remote server has an updated version, please run `cnvrg data download` first or cnvrg data upload --new_branch", Thor::Shell::Color::BLUE)
-      #       exit(1)
-      #     end
-      #   check = Helpers.checkmark()
-
-
-      #   commit_time = result["result"]["commit_time"]
-      #   already_uploaded = result["result"]["uploaded"]
-      #   dirs_uploaded = []
-      #   files_uploaded = []
-      #   files_uploaded_names = []
-      #   if !already_uploaded.nil? and !already_uploaded.empty?
-
-      #     dirs_uploaded = already_uploaded["dirs"]
-      #     files_uploaded = already_uploaded["blobs"]
-      #     if !files_uploaded.nil? and !files_uploaded.empty?
-      #       files_uploaded_names = already_uploaded["blobs"].keys
-      #     end
-
-      #   end
-
-      #   result = result["result"]["tree"]
-      #   if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-      #     @files.delete_commit(next_commit)
-
-      #     log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, true)
-      #     return false
-      #   end
-      #   log_message("Creating new commit for dataset: #{@dataset.slug}", Thor::Shell::Color::BLUE) unless  options[:sync]
-
-
-      #   # upload / update
-      #   begin
-
-      #     parallel_options = {
-      #         :progress => {
-      #             :title => "Upload Progress",
-      #             :progress_mark => '=',
-      #             :format => "%b>>%i| %p%% %t",
-      #             :starting_at => 0,
-      #             :total => (result["added"] + result["updated_on_local"]).size,
-      #             :autofinish => true
-      #         },
-      #         in_threads: ParallelThreads,
-      #         isolation: true
-      #     }
-      #     successful_updates = []
-      #     update_count =0
-      #     update_total = (result["added"] + result["updated_on_local"] + result["deleted"]).size
-
-      #     if (result["added"] + result["updated_on_local"]).size > 0
-      #       begin
-      #         upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
-      #           absolute_path = "#{@dataset.local_path}/#{f}"
-      #           relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-      #           if File.directory?(absolute_path)
-      #             if dirs_uploaded.include? f
-      #               #Already uploaded and didn't finish commit
-      #               log_message("Already uploaded dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-      #               next
-      #             end
-      #             log_message("uploading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-      #             resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-      #             if resDir
-      #               update_count += 1
-      #               successful_updates<< relative_path
-      #             else
-      #               log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
-
-      #               raise Parallel::Kill
-      #             end
-
-      #           else
-      #             if files_uploaded_names.include? f
-      #               log_message("Already uploaded file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-      #               next
-      #             end
-      #             log_message("uploading: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-      #             res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-      #             if res
-      #               update_count += 1
-
-      #               successful_updates<< relative_path
-      #             else
-      #               log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
-
-      #               raise Parallel::Kill
-
-      #             end
-      #           end
-      #         end
-      #       rescue SignalException
-      #         log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-
-
-      #         exit(1)
-      #       end
-      #     end
-
-      #   end
-      #     successful_updates = upload_result.to_a + dirs_uploaded + files_uploaded_names
-      #   successful_deletions =[]
-
-
-      #     # delete
-
-      #     deleted = update_deleted(result["deleted"])
-      #     begin
-
-      #       deleted_result = Parallel.map(deleted, in_threads: ParallelThreads) do |f|
-
-      #         relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-      #         if relative_path.end_with?("/")
-      #           log_message("deleting dir: #{f}", Thor::Shell::Color::RED, options["verbose"])
-
-      #           if @files.delete_dir(f, relative_path, commit_sha1)
-      #             successful_deletions << f
-      #           else
-      #             log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
-      #             raise Parallel::Kill
-
-      #           end
-      #         else
-      #           log_message("deleteing file: #{f}", Thor::Shell::Color::RED, options["verbose"])
-
-      #           if @files.delete_file(f, relative_path, commit_sha1)
-      #             successful_deletions << f
-
-      #           else
-      #             log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
-      #             raise Parallel::Kill
-
-
-      #           end
-      #         end
-      #       end
-
-
-      #     successful_deletions += successful_deletions.select { |x| not x.nil? }
-
-
-      #     successful_updates = successful_updates.select { |x| not x.nil? }
-
-      #     update_count = successful_updates.size
-
-      #   rescue SignalException
-      #     say "User aborted", Thor::Shell::Color::RED
-      #     exit(0)
-      #   rescue => e
-      #     log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-      #     log_error(e)
-      #     exit(0)
-      #   end
-      #   if !result["deleted"].nil? and !result["deleted"].empty?
-      #     update_count += result["deleted"].size
-      #   end
-
-      #   if update_count == update_total
-      #     begin
-      #       @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
-
-      #       return true
-      #     rescue => e
-      #       log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-
-      #       exit(1)
-
-      #     end
-
-      #   end
-      # rescue => e
-
-      #   log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
-      #   log_error(e)
-
-      #   exit(1)
-      # rescue SignalException
-
-      #   say "\nAborting", Thor::Shell::Color::BLUE
-      #   exit(1)
-      # end
-
+      force = options[:force] || false
+      chunk_size = options[:chunk_size]
+      chunk_size = [chunk_size, 1].max
+      new_branch = options["new_branch"] || false
+      new_tree = {}
+      force = options["force"] || false
+      parallel_threads = options["parallel"] || ParallelThreads
+      all_files = @dataset.list_all_files
+      progressbar = ProgressBar.create(:title => "Upload Progress",
+                                       :progress_mark => '=',
+                                       :format => "%b>>%i| %p%% %t",
+                                       :starting_at => 0,
+                                       :total => all_files.length,
+                                       :autofinish => true)
+      files_uploaded = 0
+      all_files.each_slice(chunk_size).each do |list_files|
+        Cnvrg::Logger.log_info("Uploading files into #{@dataset.slug}, #{files_uploaded} files uploaded")
+        temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
+        upload_resp = @files.upload_multiple_files(new_commit, temp_tree,
+                                                   threads: parallel_threads,
+                                                   force: force,
+                                                   new_branch: new_branch,
+                                                   progressbar: progressbar,
+                                                   partial_commit: partial_commit)
+        files_uploaded += upload_resp
+        temp_tree.each do |k, v|
+          new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
+        end
+      end
+      @dataset.write_tree(new_tree) #we dont want to re-run it every time so just on finish.
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+      return files_uploaded
     end
 
 
     desc 'upload', 'Upload updated files'
     method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
     method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--new_branch"], :desc => "create new branch of commits"
+    method_option :in_exp, :type => :boolean, :aliases => ["-ie", "--in-exp"], :desc => "In Experiment"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -2446,7 +2250,7 @@ module Cnvrg
     method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false,output_dir="output",job_type=nil,job_slug=nil )
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil)
       begin
         # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
@@ -2457,7 +2261,7 @@ module Cnvrg
           commit_msg = ""
         end
         return_id = options["return_id"]
-        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: get_project_home)
         ignore = options[:ignore] || ""
         force = options[:force] || force || false
         spec_files_to_upload = options["files"]
@@ -2467,19 +2271,19 @@ module Cnvrg
           spec_files_to_upload = spec_files_to_upload.split(",")
         end
         if @project.is_git
-        git_output_dir = options["output_dir"] || output_dir
-        if git_output_dir.present?
-          if git_output_dir.ends_with? "/"
-            git_output_dir = git_output_dir[0..-2]
-          end
-          list  = @project.generate_output_dir(output_dir)
-          spec_files_to_upload  = list
-          if spec_files_to_upload.blank?
-            log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+          git_output_dir = options["output_dir"] || output_dir
+          if git_output_dir.present?
+            if git_output_dir.ends_with? "/"
+              git_output_dir = git_output_dir[0..-2]
+            end
+            list = @project.generate_output_dir(git_output_dir)
+            spec_files_to_upload = list
+            if spec_files_to_upload.blank?
+              log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
               return true
+            end
+            force = true
           end
-          force = true
-        end
         end
 
         if ignore.nil? or ignore.empty?
@@ -2498,7 +2302,7 @@ module Cnvrg
         end
         new_branch = options["new_branch"] || @project.is_branch
 
-        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"],in_exp:in_exp, specific_files: spec_files_to_upload)
+        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"], in_exp: in_exp, specific_files: spec_files_to_upload)
         commit = result["result"]["commit"]
 
         if !link
@@ -2511,7 +2315,12 @@ module Cnvrg
         end
         result = result["result"]["tree"]
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+          msg = "#{check} Project is up to date"
+          if return_id
+            Cnvrg::Logger.jsonify(msg: msg, success: true)
+          else
+            log_message(msg, Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+          end
           return true
         end
         update_count = 0
@@ -2537,233 +2346,78 @@ module Cnvrg
         end
         job_type = options['job_type'] || job_type
         job_slug = options['job_slug'] || job_slug
-        commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit:exp_start_commit,
-                                          job_type: job_type, job_slug: job_slug, start_commit: current_commit)["result"]["commit_sha1"]
+        commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit: exp_start_commit,
+                                          job_type: job_type, job_slug: job_slug, start_commit: current_commit, message: options["message"])["result"]["commit_sha1"]
         # upload / update
-        begin
-          parallel_options = {
-              :progress => {
-                  :title => "Upload Progress",
-                  :progress_mark => '=',
-                  :format => "%b>>%i| %p%% %t",
-                  :starting_at => 0,
-                  :total => (result["added"] + result["updated_on_local"]).size,
-                  :autofinish => true
-              },
-              in_processes: ParallelProcesses,
-              in_thread: ParallelThreads,
-              isolation: true
-          }
-          if (result["added"] + result["updated_on_local"]).size > 0
-
-            begin
-              upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
-                absolute_path = "#{@project.local_path}/#{f}"
-                relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-                if File.directory?(absolute_path)
-                  log_message("uploading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-                  resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-                  if resDir
-                    f
-                    # progressbar.increment
-                    update_count += 1
-                    successful_updates << relative_path
-                  else
-                    log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
-
-                    raise Parallel::Kill
-                  end
-
-                else
-                  log_message("uploading: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-                  res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-                  if res
-                    f
-                    update_count += 1
-
-                    successful_updates << relative_path
-                  else
-                    log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
-
-                    raise Parallel::Kill
-
-                  end
-                end
-              end
-            rescue SignalException
-              log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
-              @files.rollback_commit(commit_sha1)
-              return false
-            end
-          end
-
-
-          successful_updates = upload_result.to_a
-
-
-          # delete
-
-          deleted = update_deleted(result["deleted"])
-          begin
-
-            deleted_result = Parallel.map(deleted, in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
-
-              relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-              if relative_path.end_with?("/")
-                log_message("deleting dir: #{f}", Thor::Shell::Color::RED, options["verbose"])
-
-                if @files.delete_dir(f, relative_path, commit_sha1)
-                  f
-                else
-                  log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
-
-                end
-              else
-                log_message("deleteing file: #{f}", Thor::Shell::Color::RED, options["verbose"])
-
-                if @files.delete_file(f, relative_path, commit_sha1)
-                  f
-                else
-                  log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
-
-                end
-              end
-            end
-          rescue Interrupt
-            log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
-            @files.rollback_commit(commit_sha1)
-
-            return false
-
-          end
-
-
-          successful_deletions += successful_deletions.select {|x| not x.nil?}
-
-
-          successful_updates = successful_updates.select {|x| not x.nil?}
-
-          update_count = successful_updates.size
-
-        rescue SignalException
-          @files.rollback_commit(commit_sha1)
-          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
-          return false
-        rescue => e
-          @files.rollback_commit(commit_sha1)
-          log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
-          log_error(e)
-          return false
-        end
-        if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count += result["deleted"].size
-        end
-
-        if update_count == update_total
-          res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
-          if (Cnvrg::CLI.is_response_success(res, false))
-            # save idx
-            begin
-              @project.update_idx_with_files_commits!((successful_deletions + successful_updates), res["result"]["commit_time"])
-
-              @project.update_idx_with_commit!(commit_sha1)
-            rescue => e
-              @files.rollback_commit(commit_sha1)
-              log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
-              log_error(e)
-              return false
-
-            end
-            image = is_project_with_docker(Dir.pwd)
-            if image and image.is_docker
-              image.update_image_activity(commit_sha1, nil)
-            end
-
-            if options["verbose"]
-              log_message("#{check} Done", Thor::Shell::Color::BLUE)
-              if successful_updates.size > 0
-                successful_updates.flatten!
-                log_message("Updated:", Thor::Shell::Color::GREEN)
-                suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-                log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
-              end
-              if successful_deletions.size > 0
-                successful_deletions.flatten!
-
-                log_message("Deleted:", Thor::Shell::Color::GREEN)
-                del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-                log_message(del.join("\n"), Thor::Shell::Color::GREEN)
-              end
-              log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
-            else
-              if return_id
-                print_res = {
-                    'success' => "true",
-                    'commit_sha1' => res["result"]["commit_id"]
-                }
-                puts JSON[print_res]
-                return JSON[print_res]
-              end
-              if (options["sync"] or sync) and direct
-                log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
-                return true
-
-              else
-                log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
-                return true
-
-              end
-
-            end
-
-          else
-            @files.rollback_commit(commit_sha1)
-            if return_id
-              print_res = {
-                  'success' => "false",
-                  'message' => 'couldn\'t commit changes,  Rolling Back all changes.'
-              }
-              puts JSON[print_res]
-            else
-              log_message("Error: couldn't commit changes,  Rolling Back all changes.", Thor::Shell::Color::RED)
-
-            end
-            return false
-          end
+        # delete
+        to_upload = result["added"] + result["updated_on_local"]
+        deleted = result["deleted"]
+        progressbar = ProgressBar.create(:title => "Upload Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => (to_upload.size + deleted.size),
+                                         :autofinish => true)
+        @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
+        @files.delete_files_from_server(deleted, commit_sha1)
+        progressbar.progress += deleted.size
+        res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
+        unless Cnvrg::CLI.is_response_success(res, false)
+          raise StandardError.new("Cant end commit")
+        end
+        # save idx
+        @project.update_idx_with_files_commits!((to_upload + deleted), res["result"]["commit_time"])
+        @project.update_idx_with_commit!(commit_sha1)
+        if options["verbose"]
+          log_message("#{check} Done", Thor::Shell::Color::BLUE)
+          if successful_updates.size > 0
+            successful_updates.flatten!
+            log_message("Updated:", Thor::Shell::Color::GREEN)
+            suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
+            log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
+          end
+          if successful_deletions.size > 0
+            successful_deletions.flatten!
+            log_message("Deleted:", Thor::Shell::Color::GREEN)
+            del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
+            log_message(del.join("\n"), Thor::Shell::Color::GREEN)
+          end
+          log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
         else
-          @files.rollback_commit(commit_sha1)
           if return_id
             print_res = {
-                'success' => "false",
-                'message' => "uploaded only: #{update_count} / #{update_total}, \n Rolling back"
-
+                'success' => "true",
+                'commit_sha1' => res["result"]["commit_id"]
             }
             puts JSON[print_res]
+            return JSON[print_res]
+          end
+          if (options["sync"] or sync) and direct
+            log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
+            return true
           else
-          log_message("Error: uploaded only: #{update_count} / #{update_total}, \n Rolling back", Thor::Shell::Color::RED)
+            log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
+            return true
           end
-          return false
-
         end
       rescue => e
-
-        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
-        log_error(e)
-        @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
-        puts e.message
-        return false
-
-      rescue SignalException
-
-        say "\nAborting", Thor::Shell::Color::BLUE
-        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        if e.is_a? SignalException
+          say "\nAborting", Thor::Shell::Color::BLUE
+          say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        else
+          log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+          log_error(e)
+        end
         @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
+        print_res = {
+            'success' => "false",
+            'message' => 'couldn\'t commit changes,  Rolling Back all changes.'
+        }
+        puts JSON[print_res] if return_id
         return false
       end
-
     end
+
     desc 'download_file_data', 'Download one data files'
     method_option :verbose, :type => :boolean, :aliases => ["-v", "--verbose"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :desc => "download specified commit", :default => nil
@@ -2848,9 +2502,11 @@ module Cnvrg
         end
         result = res["result"]["tree"]
         commit = res["result"]["commit"]
+        can_commit = res["result"]["can_commit"] || false #can commit means that our next commit is newer than the latest commit
+        # so if can_commit is true it means that we have to be up-to-date with the dataset.
         update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
         successful_changes  = 0
-        if update_total == 0
+        if update_total == 0 or can_commit
           log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
           return true
         elsif options["verbose"]
@@ -2858,6 +2514,8 @@ module Cnvrg
         else
           log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
         end
+        Cnvrg::Logger.log_info("Current commit: #{@dataset.get_current_commit}, destination commit: #{commit}")
+        Cnvrg::Logger.log_info("Compare idx res: #{result}")
         conflicts = @files.mark_conflicts(result)
         
         log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
@@ -2887,12 +2545,13 @@ module Cnvrg
           end
           return true          
         end
-      rescue SignalException
+      rescue SignalException => e
+        Cnvrg::Logger.log_error(e)
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       rescue => e
         log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
-        log_error(e)
+        Cnvrg::Logger.log_error(e)
         exit(1)
       end
     end
@@ -2949,7 +2608,7 @@ module Cnvrg
         log_start(__method__, args, options)
         project_home = get_project_home
         @project = Project.new(project_home)
-        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug, project_home: project_home)
         git = options["git"]
         commit = options["commit"]
         if git or @project.is_git
@@ -2971,174 +2630,89 @@ module Cnvrg
           log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
         new_branch = options["new_branch"] || @project.is_branch || false
-        res = @project.compare_idx(new_branch, commit: nil, in_exp: in_exp)["result"]
+        res = @project.compare_idx(new_branch, in_exp: in_exp, download: true)["result"]
         result = res["tree"]
 
         commit = res["commit"]
-        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
+
+        #here im grouping the files by their current status.
+        #
+        # all the files that changed in the server (added + updated)
+        changed_files = result['updated_on_server'] + result['added']
+
+        # non-conflicted files, all the files that changed remotely but not locally
+        updated_files = changed_files - result["update_local"]
+        # conflicted - files that changed remotely and locally
+        conflicted_files = changed_files & result["update_local"]
+
+        # all deleted files
+        all_deleted = result['deleted']
+        # cant delete files - files that deleted on the server but changed locally
+        conflicted_deleted = all_deleted & result["update_local"]
+        # files to delete - files that deleted on the server and unchanged locally
+        deleted_files = all_deleted - conflicted_deleted
+
+
+        update_total = [all_deleted, changed_files ].flatten.size
+
+        if update_total < 1
           if !@project.last_local_commit.eql? commit
+            Cnvrg::Logger.log_info("Finish commit, updating idx with commit")
             @project.update_idx_with_commit!(commit)
           end
           log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
           return true
         end
-        update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
-
+        Cnvrg::Logger.log_info("Got #{update_total} changes from server")
 
         successful_changes = []
         if update_total == 1
           log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !options["sync"])
-        elsif update_total == 0
-          log_message("Project is up to date", Thor::Shell::Color::GREEN, !options["sync"])
-          return true
         elsif options["verbose"]
           log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
         else
           log_message("Syncing files", Thor::Shell::Color::BLUE, !options["sync"])
-
         end
-        parallel_options = {
-            :progress => {
-                :title => "Download Progress",
-                :progress_mark => '=',
-                :format => "%b>>%i| %p%% %t",
-                :starting_at => 0,
-                :total => result["updated_on_server"].size,
-                :autofinish => true
-            },
-            in_processes: ParallelProcesses,
-            in_thread: ParallelThreads
-        }
-        if !result["conflicts"].empty?
-          begin
-
-
-            conflicts_result = Parallel.map(result["conflicts"], in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
-
-              relative_path = f.gsub(/^#{@project.local_path}/, "")
-              log_message("downloading: #{f}.conflict", Thor::Shell::Color::BLUE, options["verbose"])
-              if relative_path.eql? ".cnvrgignore"
-                if @files.download_file_s3(f, relative_path, project_home, commit_sha1 = nil, conflict = false)
-                  f
-                else
-                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
-                end
-              else
-                if @files.download_file_s3(f, relative_path, project_home, commit_sha1 = nil, conflict = true)
-                  f
-                else
-                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
-
-                end
-              end
-            end
-          rescue Interrupt
-
-            log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
-
-            @files.revoke_download(result["conflicts"], [])
-            exit(1)
-          end
-        end
-
-
-        successful_changes += conflicts_result.to_a
-        if !result["updated_on_server"].empty?
-          begin
-            updated_on_server_result = Parallel.map(result["updated_on_server"], parallel_options) do |f|
-
-              relative_path = f.gsub(/^#{@project.local_path}/, "")
-              if f.end_with? "/"
-                # dir
-                log_message("downloading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-
-                if @files.download_dir(f, relative_path, project_home)
-                  f
-                else
-                  log_message("Couldn't create directory: #{f}", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
-
-
-                end
-
-              else
-                # blob
-                log_message("downloading file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
-                if @files.download_file_s3(f, relative_path, project_home)
-                  f
-                else
-
-
-                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
-                  raise Parallel::Kill
-
-
-                end
-              end
-
-
-            end
-            successful_changes += updated_on_server_result.to_a
-          rescue Interrupt
-            log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
-
-            @files.revoke_download(result["conflicts"], result["updated_on_server"])
-            exit(1)
-
-          end
-        end
-
-        deleted = result["deleted"].to_a
-        delete_res = @files.delete_commit_files_local(deleted)
-        if !delete_res
-          log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
-          log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
-
-          @files.revoke_download(result["conflicts"], result["updated_on_server"])
-          exit(1)
-
-        end
-        successful_changes += deleted
 
+        progressbar = ProgressBar.create(:title => "Download Progress",
+                                         :progress_mark => '=',
+                                         :format => "%b>>%i| %p%% %t",
+                                         :starting_at => 0,
+                                         :total => update_total,
+                                         :autofinish => true)
 
-        successful_changes = successful_changes.select {|x| not x.nil?}
 
-        if update_total == successful_changes.size
-          # update idx with latest commit
-          @project.update_idx_with_commit!(commit)
-          check = Helpers.checkmark()
+        Cnvrg::Logger.log_info("Downloading updated files:#{updated_files.join(",")}")
+        @files.download_files(updated_files, commit, progress: progressbar) if updated_files.present?
 
-          if options["verbose"]
-            log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
-            log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
-            log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
-          else
-            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
-          end
+        Cnvrg::Logger.log_info("Downloading conflicted files:#{conflicted_files.join(",")}")
+        @files.download_files(conflicted_files, commit, postfix: ".conflict", progress: progressbar) if conflicted_files.present?
 
+        Cnvrg::Logger.log_info("Delete files: #{deleted_files.join(",")}")
+        @files.delete_files_local(deleted_files, conflicted: conflicted_deleted, progress: progressbar)
 
+        # update idx with latest commit
+        @project.update_idx_with_commit!(commit)
+        #TODO Sync, remove idx, sync again and pray
+        check = Helpers.checkmark()
+        Cnvrg::Logger.log_info("Finished downloading successfuly")
+        if options["verbose"]
+          log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
+          log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
+          log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
+        else
+          log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
         end
       rescue => e
-
         log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
-        log_error(e)
-        if successful_changes.nil?
-          exit(1)
-        end
-        begin
-          @files.revoke_download(result["conflicts"], result["updated_on_server"])
-        end
+        Cnvrg::Logger.log_error(e)
+        exit(1)
+      rescue Exception => e
+        Cnvrg::Logger.log_error(e)
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         exit(1)
       rescue SignalException
-        say "\nAborting", Thor::Shell::Color::BLUE
-        if successful_changes.nil?
-          exit(1)
-        end
-        begin
-          @files.revoke_download(result["conflicts"], result["updated_on_server"])
-        end
+        log_message("\nAborting", Thor::Shell::Color::BLUE)
         exit(1)
       end
     end
@@ -3168,7 +2742,24 @@ module Cnvrg
         resolver = resp['result']['resolver']
         latest = resp['result']['latest']
         commit = resp['result']['commit']
-        @files.handle_compare_idx(compare, resolver: resolver)
+        updated_files = compare['updated_on_server']
+        conflicted_files = compare['conflicts']
+        deleted_files = compare['deleted']
+        overall_changes = [updated_files, conflicted_files, deleted_files].flatten.size
+
+
+        progressbar = ProgressBar.create(:title => "Download Progress",
+                           :progress_mark => '=',
+                           :format => "%b>>%i| %p%% %t",
+                           :starting_at => 0,
+                           :total => overall_changes,
+                           :autofinish => true)
+
+        @files.download_files(updated_files, commit_sha1, progress: progressbar)
+        @files.download_files(conflicted_files, commit_sha1, progress: progressbar, postfix: '.conflicted')
+        @files.delete_files_local(deleted_files, progress: progressbar)
+
+
         @project.set_on_branch(latest)
         @project.update_idx_with_commit!(commit)
         @project.generate_idx
@@ -3326,9 +2917,10 @@ module Cnvrg
       job_slug = options['job_slug']
       job_type = options['job_type']
       in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
+      in_exp = false if job_type.present? and job_type == "NotebookSession"
       run_download = true
       if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp
-        run_download =false
+        run_download = false
       end
       if run_download
         invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
@@ -3336,8 +2928,6 @@ module Cnvrg
       invoke :upload, [false, true,  direct, "",in_exp,options[:force],  options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
              :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
-
-
     end
 
     desc 'run cmd', 'Runs an experiment'
@@ -3371,7 +2961,7 @@ module Cnvrg
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :git_commit, :type => :string, :aliases => [ "--git_commit"], :default => nil
     method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
-    method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart"], :default => nil
+    method_option :restart_if_stuck, :type => :boolean, :aliases => ["--restart","--restart_if_stuck"], :default => nil
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
 
     def run(*cmd)
@@ -3413,6 +3003,7 @@ module Cnvrg
       git_branch = options["git_branch"]
       restart_if_stuck = options["restart_if_stuck"]
 
+
       options_hash = Hash[options]
 
       if local
@@ -3786,6 +3377,7 @@ module Cnvrg
     method_option :local_folders, :type => :string, :aliases => ["--local_folders"], :default => nil
 
     def exec_remote(*cmd)
+
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
@@ -3852,7 +3444,7 @@ module Cnvrg
         git_branch = options["git_branch"]
         options_hash = Hash[options]
         local_folders_options = options["local_folders"]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit",
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
                              "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders" )
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
@@ -5909,33 +5501,15 @@ module Cnvrg
           say message, type
         end
         begin
-          if type == Thor::Shell::Color::BLUE
-            $LOG.info message: message, type: "info"
-          elsif type == Thor::Shell::Color::RED
-            $LOG.error message: message, type: "error"
-          elsif type == Thor::Shell::Color::YELLOW
-            $LOG.warn message: message, type: "warning"
-          elsif type == Thor::Shell::Color::GREEN
-            $LOG.info message: message, type: "success"
-          else
-            $LOG.info message: message, type: "unknown"
-          end
-
+          Cnvrg::Logger.log_info(message)
         rescue => e
-          puts e.backtrace
+          Cnvrg::Logger.log_error(e)
         end
       end
 
       def log_error(e)
         begin
-          $LOG.error message: e.message, type: "error"
-          size = e.backtrace.size
-          min = 10
-          if min >= size
-            min = size
-          end
-          $LOG.error message: "Error: #{e.message}, Backtrace: #{e.backtrace[0..min].join(",")}", type: "error"
-        rescue
+          Cnvrg::Logger.log_error(e)
         end
       end
 
@@ -5952,6 +5526,7 @@ module Cnvrg
       end
 
       def self.is_response_success(response, should_exit = true)
+
         begin
           if response.nil? or !response
             # if !Cnvrg::Helpers.internet_connection?
@@ -5965,7 +5540,7 @@ module Cnvrg
               return false
             end
           elsif response["status"] != 200
-            error = response['message']
+            error = response['message'] || "Unknown error"
             # Cnvrg::CLI.log_end(1, error)
             if response["status"] == 500
               say("<%= color('Server Error', RED) %>")
@@ -6027,6 +5602,10 @@ module Cnvrg
       end
 
       def get_project_home
+        return Cnvrg::CLI.get_project_home
+      end
+
+      def self.get_project_home
         absolute_path = Dir.pwd
         dirs = absolute_path.split("/")
         dirs.pop while not Dir.exists?("#{dirs.join("/")}/.cnvrg") and dirs.size != 0
@@ -6093,24 +5672,9 @@ module Cnvrg
           $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
           remove_old_log_files()
         rescue
-
         end
       end
 
-      def remove_old_log_files()
-        begin
-          last_week = (Time.now - (7 * 24 * 60 * 60)).strftime("%Y-%m-%d")
-          home = File.expand_path('~')
-          log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
-          log_files.each do |l|
-            if File.mtime(l).strftime("%Y-%m-%d") < last_week
-              FileUtils.rm_rf(l)
-            end
-          end
-        end
-
-      end
-
         def verify_logged_in(in_dir=true)
           begin
           log_handler()