cnvrg 0.3.6 → 0.3.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3f314a2eaa98e7dc70b1c2cf84cc935921a0601c
-  data.tar.gz: bdd778e228e5d73f652843b2aae7ab6dfe6a4a54
+  metadata.gz: 14812c074d9b241c639f215c82da967e632a5b4c
+  data.tar.gz: 68845b081b102ec1d06499a09e3e576a00a89e65
 SHA512:
-  metadata.gz: 163f6104da853f34810b685a738be1caae9d7f4da348c6c971b30daeb12360a46f177600d592db47fdc99bdaed85b98d9db2fb3a2f0a6a3327683e3d4b9f5a7e
-  data.tar.gz: e104de7d5bd6b151f2bcdffb43f17537ecbaf4b97e53d923a15076e1eaabf8d90df1b808f67c6c1b2609a5a6d70366eb36fcf00b74583bbf1eccccab48c6ef9f
+  metadata.gz: ce3a5373033f6a01bab5e0eb9d839e3a6a3b7d1b15576a996e1c00e5d6ef091e9ac6ac50a320379aa9f7ac691b16f8caacc3116b5bd79c4f3c464fbf17878c23
+  data.tar.gz: 2d1ece161322402bd3922b52f0e0371577eeade4c650d037c7956d13c04a3ce701647ff45f45c4c5415006b6e0a6c33f12bda2ddbbb9103c0a0802a49b3438d6

data/cnvrg.gemspec

@@ -44,6 +44,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'activesupport', '~> 5.0'
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'net-ssh'
+    spec.add_runtime_dependency 'down'
 
 
 

data/lib/cnvrg/api.rb

@@ -44,24 +44,33 @@ module Cnvrg
 
             @user, @pass = n[Cnvrg::Helpers.netrc_domain]
             begin
-            conn = Faraday.new
+                if !Helpers.is_verify_ssl
+
+                    conn = Faraday.new "#{endpoint_uri}", :ssl => {:verify => false}
+                else
+                    conn = Faraday.new "#{endpoint_uri}"
+                end
             conn.headers['Auth-Token'] = @pass
             conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
-
             case method
-            when 'GET'
-                response = conn.get "#{endpoint_uri}/#{resource}"
-
+                when 'GET'
+                response = conn.get "#{resource}", data
+                if response.to_hash[:status] == 404
+                  return false
+                end
                 if parse_request == true
                     JSON.parse(response.body)
                 else
                     response
                 end
-              when 'POST'
+                when 'POST'
                 conn.options.timeout = 420
                 conn.options.open_timeout =420
-                response = conn.post "#{endpoint_uri}/#{resource}", data
+                response = conn.post "#{resource}", data
 
+                if response.to_hash[:status] == 404
+                  return false
+                end
                 if parse_request == true
                     JSON.parse(response.body)
                 else
@@ -72,10 +81,14 @@ module Cnvrg
                     fr.headers['Auth-Token'] = @pass
                     fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
                     fr.headers["Content-Type"] = "multipart/form-data"
+                    if !Helpers.is_verify_ssl
+	                      fr.ssl.verify = false
+                    end
 
 
                     fr.request :multipart
                     fr.request :url_encoded
+                    fr.request :retry, max: 2, interval: 0.05,interval_randomness: 0.5, backoff_factor: 2
                     fr.adapter :net_http
                 end
                 conn.options.timeout = 420
@@ -100,6 +113,9 @@ module Cnvrg
                 response = conn.post "#{endpoint_uri}/#{resource}", data
 
                 FileUtils.rm_rf(temp_path)
+                if response.to_hash[:status] == 404
+                  return false
+                end
 
 
                 if parse_request == true
@@ -109,7 +125,9 @@ module Cnvrg
                 end
             when 'DELETE'
                 response = conn.delete "#{endpoint_uri}/#{resource}", data
-
+                if response.to_hash[:status] == 404
+                  return false
+                end
                 if parse_request == true
                     JSON.parse(response.body)
                 else

data/lib/cnvrg/auth.rb

@@ -51,6 +51,11 @@ module Cnvrg
 
             if url.scheme == 'https'
                 http.use_ssl = true
+                if !Helpers.is_verify_ssl
+                    http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+                end
+
+
             end
             req = Net::HTTP::Post.new(url.request_uri)
 

data/lib/cnvrg/cli.rb

@@ -60,6 +60,7 @@ class Thor
 
       end
 
+
       # Let Thor::Options parse the options first, so it can remove
       # declared options from the array. This will leave us with
       # a list of arguments that weren't declared.
@@ -193,12 +194,16 @@ module Cnvrg
     map %w(-v --version) => :version
 
     desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
+    method_option :verify_ssl, :type => :boolean, :aliases => ["-s", "--verify_ssl"], :default => true
 
     def set_api_url(url)
+      log_handler()
+      log_start(__method__, args, options)
       home_dir = File.expand_path('~')
       if !url.end_with? "/api"
         url = url + "/api"
       end
+      verify_ssl = options["verify_ssl"]
       begin
         if !File.directory? home_dir + "/.cnvrg"
           FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
@@ -207,42 +212,34 @@ module Cnvrg
           FileUtils.touch [home_dir + "/.cnvrg/config.yml"]
         end
         compression_path = "#{File.expand_path('~')}/.cnvrg/tmp"
-
         begin
-          config = YAML.load_file(home_dir + "/.cnvrg/config.yml")
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
           if !config
-            config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path, }
-
+            config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path ,verify_ssl: verify_ssl }
           end
+
+
         rescue
-          config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path}
+          config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path ,verify_ssl: verify_ssl }
         end
-        owner = config.to_h[:owner]
 
         say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
         if config.empty?
-          config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path}
+          config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path, verify_ssl: verify_ssl }
         else
           if !config.to_h[:compression_path].nil?
             compression_path = config.to_h[:compression_path]
           end
-          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
+          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path, verify_ssl: verify_ssl}
         end
-        # res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
-        # if Cnvrg::CLI.is_response_success(res, false)
 
         checks = Helpers.checkmark
+        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
 
-
-        File.open(home_dir + "/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
         say "#{checks} Done", Thor::Shell::Color::GREEN
-          # else
-          #   say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
-          #   exit(1)
-          #
-          # end
 
       rescue => e
+        log_error(e)
         say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
       end
     end
@@ -272,6 +269,16 @@ module Cnvrg
         say "Done"
       rescue
         say "ERROR", Thor::Shell::Color::RED
+          File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          say "#{checks} Done", Thor::Shell::Color::GREEN
+        # else
+        #   say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
+        #   exit(1)
+        #
+        # end
+
+      rescue => e
+        say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
       end
     end
 
@@ -553,7 +560,7 @@ module Cnvrg
 
     def new(project_name)
       begin
-        verify_logged_in(false)
+       verify_logged_in(false)
         log_start(__method__, args, options)
         clean = options["clean"]
         docker_image = options["docker_image"]
@@ -568,8 +575,8 @@ module Cnvrg
           exit(1)
 
         end
-        if Project.create(project_name, clean, with_docker = docker)
-          path = Dir.pwd + "/" + project_name
+        if Project.create(project_name, clean, with_docker=docker)
+	       path = Dir.pwd + "/" + project_name
           @project = Project.new(path)
           @project.generate_idx
           if docker
@@ -635,42 +642,6 @@ module Cnvrg
       end
     end
 
-    desc 'set_image', 'set_image for a project'
-
-    def set_image(docker_image)
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      working_dir = is_cnvrg_dir
-      local_images = Docker::Image.all
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.include? docker_image}.flatten
-      if docker_image_local.size == 0
-
-        if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-          image = pull(docker_image)
-          if image
-            say "downloaded image: #{docker_image}"
-            @image = Images.new(working_dir, docker_image)
-          else
-            say "Could not set image, image was not found", Thor::Shell::Color::RED
-            exit(1)
-          end
-        else
-          say "Could nset image, image was not found", Thor::Shell::Color::RED
-          exit(1)
-
-        end
-      elsif docker_image_local.size == 1
-        say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
-        @image = Images.new(working_dir, docker_image_local[0])
-      elsif docker_image_local.size > 1
-        say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
-        image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-        image_name = image_name.strip
-        @image = Images.new(working_dir, image_name)
-      end
-      @image.update_image_activity(nil, nil)
-
-    end
 
     desc 'link', 'Link current directory to a new cnvrg project'
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
@@ -793,24 +764,51 @@ module Cnvrg
         exit(1)
       end
     end
+    desc 'data delete', 'delete dataset'
+    def delete_data(dataset_slug)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+          owner = CLI.get_owner
+        response = Dataset.delete(dataset_slug, owner)
 
-    desc 'data clone', 'Clone dataset'
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
-    method_option :remote, :type => :boolean, :aliases => ["-r", "--remote"], :default => false
+        if Cnvrg::CLI.is_response_success(response)
+          log_message("Successfully deleted dataset: #{dataset_slug}", Thor::Shell::Color::GREEN)
+        else
+          log_message("Error while tying to delete dataset: #{response["messages"]}", Thor::Shell::Color::RED)
 
 
-    def clone_data(dataset_url)
+        end
+
+      rescue => e
+        log_error(e)
+      rescue SignalException
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'data clone', 'Clone dataset'
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
+    method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
+    def clone_data(dataset_url,only_tree=false,commit=nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
+        commit = options["commit"] || commit
+        only_tree = options["only_tree"] || only_tree
+
+
         url_parts = dataset_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
-        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
-
-        Cnvrg::CLI.is_response_success(response)
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit})
+        Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["result"]["name"]
+        dataset_home = Dir.pwd+"/"+dataset_name
+
 
         if (Dir.exists? dataset_name)
           log_message("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
@@ -822,35 +820,88 @@ module Cnvrg
         end
         if Dataset.clone(owner, dataset_name, slug)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
+          @files = Cnvrg::Datafiles.new(owner, slug)
 
-          commit_to_clone = options["commit"] || nil
-          working_dir = "#{Dir.pwd}/#{dataset_name}"
-          @dataset = Dataset.new(working_dir)
-          @dataset.generate_idx()
+          successful_changes = []
+          log_message("Downloading files", Thor::Shell::Color::BLUE)
+          commit = response["result"]["commit"]
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(dataset_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
+          if !response["result"]["tree"].nil?
+            parallel_options = {
+                :progress => {
+                    :title => "Download Progress",
+                    :progress_mark => '=',
+                    :format => "%b>>%i| %p%% %t",
+                    :starting_at => 0,
+                    :total => response["result"]["tree"].size,
+                    :autofinish => true
+                },
+                in_threads: ParallelThreads
+            }
+            begin
+              is_success = true
 
+              Parallel.map((response["result"]["tree"]), parallel_options) do |f|
 
-          download_res = download_data(false, false, path = working_dir, in_dir = false)
-          if !download_res
-            exit(1)
-          end
-          Dataset.verify_cnvrgignore_exist(dataset_name, false)
+                relative_path = f[0].gsub(/^#{dataset_home}/, "")
+                if f[0].end_with? "/"
+                  # dir
+                  if @files.download_dir(dataset_home, relative_path)
+                    successful_changes << relative_path
+                  else
+                    is_success =false
+                    log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
+                    raise Parallel::Kill
+                  end
+                else
+                  # blob
+                  if !only_tree
+
+                  if @files.download_file_s3(f[0], relative_path, dataset_home, false, commit_sha1=commit)
+                    successful_changes << relative_path
+                  else
+                    is_success =false
+                    log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
+                    raise Parallel::Kill
+                  end
+                  end
+                end
+              end
+            rescue Interrupt
+              is_success =false
+              log_message("Couldn't download", Thor::Shell::Color::RED)
+
+              exit(1)
+            end
 
 
+          end
           check = Helpers.checkmark
-          log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+
+          successful_changes = response["result"]["tree"]
+          if !successful_changes.nil? and is_success
+            Dataset.verify_cnvrgignore_exist(dataset_name, false)
+
+            log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+          else
+            log_message("Couldn't download some files", Thor::Shell::Color::RED)
+
+          end
 
         else
-          @dataset.revert(working_dir) unless @dataset.nil?
-          log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
-          exit(0)
+
+          log_message("Error: Couldn't create directory: #{dataset_name}", Thor::Shell::Color::RED)
+          exit(1)
         end
-      rescue => e
-        log_error(e)
       rescue SignalException
-
         say "\nAborting"
         exit(1)
       end
+
+
+
+
     end
 
     desc 'init_data_container', 'Init dataset directory', :hide => true
@@ -1029,6 +1080,8 @@ module Cnvrg
     end
 
 
+
+
     desc 'upload_data', 'Upload data files', :hide => true
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
@@ -1152,7 +1205,7 @@ module Cnvrg
           update_count += result["deleted"].size
         end
         if update_count == update_total
-          res = @files.end_commit(commit_sha1)
+          res = @files.end_commit(commit_sha1,false)
           if (Cnvrg::CLI.is_response_success(res, false))
             # save idx
             begin
@@ -1280,9 +1333,8 @@ module Cnvrg
             relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
             successful_updates << relative_path
           end
-          @dataset.update_idx_with_files_commits!((successful_deletions + successful_updates), commit_time)
+          @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
 
-          @dataset.update_idx_with_commit!(commit_sha1)
           log_message("Compressing data", Thor::Shell::Color::BLUE)
 
           home_dir = File.expand_path('~')
@@ -1366,10 +1418,10 @@ module Cnvrg
           exit(0)
         rescue => e
           log_error(e)
-          if !Cnvrg::Helpers.internet_connection?
-            say "Seems there is no internet connection", Thor::Shell::Color::RED
-
-          end
+          # if !Cnvrg::Helpers.internet_connection?
+          #   say "Seems there is no internet connection", Thor::Shell::Color::RED
+          #
+          # end
           if File.exist? log_file
             @files.upload_data_log_file(log_file, log_file, commit_sha1)
           end
@@ -1428,12 +1480,16 @@ module Cnvrg
       if owner.nil? or owner.empty?
         owner = CLI.get_owner()
       end
+
       result = @dataset.list(owner)
-      list = result["result"]["list"]
+      Cnvrg::CLI.is_response_success(result)
 
-      print_table(list)
+        list = result["result"]["list"]
 
-    end
+        print_table(list)
+
+
+      end
 
     desc 'data commits', 'List all commits for a specific dataset'
 
@@ -1640,7 +1696,7 @@ module Cnvrg
         new_branch = options["new_branch"] || false
         force = options["force"] || false
 
-        result = @project.compare_idx(new_branch, force: force)["result"]
+        result = @project.compare_idx(new_branch,force:force)["result"]
         commit = result["commit"]
         result = result["tree"]
         log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE)
@@ -1710,112 +1766,487 @@ module Cnvrg
         exit(1)
       end
     end
+    desc 'sync_data_new', 'sync_data_new'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
 
+    def sync_data_new(new_branch, force, verbose, commit, all_files)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
+      if !options[:force]
+        invoke :download_data_new,[verbose,true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
+      end
+      invoke :upload_data_new,[new_branch,  verbose,true,force], :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true
 
-    desc 'upload', 'Upload updated files'
-    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    end
+    desc 'upload_data_new', 'upload_data_new'
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
-    method_option :message, :type => :string, :aliases => ["-m", "--message"], :default => ""
-    method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
+    def upload_data_new(new_branch,  verbose,sync,force)
+      commit = invoke :start_commit_data,[], :new_branch=>new_branch, :direct=>false, :force =>force
+
+      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync
+      if upload_res
+        invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force
+      end
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "")
+    end
+
+    desc 'start_commit', 'start data commit'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :direct, :type => :boolean, :aliases => ["-d","--direct"], :desc => "was called directed", :default => true
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
 
+    def start_commit_data()
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        direct = options[:direct]
+        new_branch = options["new_branch"] || false
+        force = options["force"] || false
+        commit_sha1 = nil
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        next_commit = @dataset.get_next_commit()
+        if !next_commit.nil? and !next_commit.empty?
+          resp = @files.get_commit(next_commit)
+          if resp["result"]["status"].eql? "new"
+            resp = @files.start_commit(new_branch,force)
+            commit_sha1 = resp["result"]["commit_sha1"]
 
-        @project = Project.new(get_project_home)
-        commit_msg = options["message"]
-        if commit_msg.nil? or commit_msg.empty?
-          commit_msg = ""
-        end
-
-        @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        ignore = options[:ignore] || ""
-        force = options[:force] || false
+            @dataset.set_next_commit(commit_sha1)
 
-        if ignore.nil? or ignore.empty?
-          ignore = ignore_list
-        end
-        data_ignore = data_dir_include()
-        if !data_ignore.nil?
-          if ignore.nil? or ignore.empty?
-            ignore = data_ignore
           else
-            ignore = "#{ignore},#{data_ignore}"
-          end
-        end
-        if !@project.update_ignore_list(ignore)
-          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
-        end
-        new_branch = options["new_branch"] || false
-
-        if options["sync"] or sync
-          new_branch_exp = @project.get_new_branch
-          if new_branch_exp
-            new_branch = new_branch_exp
-          end
-        end
 
-        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"])
-        commit = result["result"]["commit"]
-        if !link
-          if ((commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?) and !force)
+              commit_sha1 = resp["result"]["commit_sha1"]
 
-            log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
-            exit(1)
           end
+        else
 
-          log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
+          resp = @files.start_commit(new_branch,force)
+          commit_sha1 = resp["result"]["commit_sha1"]
+
+          @dataset.set_next_commit(commit_sha1)
         end
-        result = result["result"]["tree"]
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "conflict"
-        #   else
-        #     num =  "conflicts"
-        #   end
-        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #
-        # end
-        check = Helpers.checkmark()
-        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
-          return true
+        if direct
+          puts commit_sha1
         end
-        update_count = 0
-        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
-        successful_updates = []
-        successful_deletions = []
-        if options["verbose"]
-          if update_total == 1
-            log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
-          else
-            log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
+        return commit_sha1
+      rescue => e
+        puts e
+          if !commit_sha1.nil?
+            @dataset.set_next_commit(commit_sha1)
           end
-        else
-          log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
+      rescue SignalException
+        log_message("Aborting..", Thor::Shell::Color::YELLOW)
+        if !commit_sha1.nil?
 
+          @dataset.set_next_commit(commit_sha1)
         end
-        # Start commit
-        commit_sha1 = @files.start_commit(new_branch, force: force)["result"]["commit_sha1"]
 
-        # upload / update
-        begin
+        exit(1)
+      end
 
-          parallel_options = {
-              :progress => {
-                  :title => "Upload Progress",
-                  :progress_mark => '=',
-                  :format => "%b>>%i| %p%% %t",
-                  :starting_at => 0,
+    end
+    desc 'end_commit', 'start data commit'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+
+    def end_commit_data(commit)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        force = options["force"] || false
+
+        resp = @files.end_commit(commit, force)
+        if (resp and resp["result"])
+          check = Helpers.checkmark()
+          @dataset.remove_next_commit()
+          log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
+          end
+
+
+      rescue => e
+        puts e
+      rescue SignalException
+        @dataset.set_next_commit(commit)
+        log_message("Aborting", Thor::Shell::Color::YELLOW)
+        exit(1)
+      end
+
+    end
+    desc 'list_files', 'list files in dataset'
+    method_option :json, :type => :boolean, :aliases => ["-j","--json"],:default => true,   :desc => "response as json"
+    method_option :commit, :type => :string, :aliases => ["-c","--commit"], :default => nil
+
+    def list_files_dataset()
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        commit = options[:commit]
+        as_json = options[:json]
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+
+        resp = @dataset.list_files(commit, as_json)
+        if (resp and resp["result"])
+          if as_json
+            puts resp["result"]
+          else
+            print_table(resp["result"])
+
+          end
+
+        end
+
+
+      rescue => e
+        puts e
+      rescue SignalException
+        @dataset.set_next_commit(commit)
+        log_message("Aborting", Thor::Shell::Color::YELLOW)
+        exit(1)
+      end
+
+    end
+    desc 'upload_data', 'Upload updated files'
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+
+    def upload_data_files(new_commit, *files)
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        next_commit = @dataset.get_next_commit()
+
+
+        if (new_commit.nil? or new_commit.empty?) and next_commit.nil?
+          log_message("You must specify commit, run start_commit to create new commit", Thor::Shell::Color::RED)
+          exit(1)
+        end
+
+        force = options[:force] || false
+
+        new_branch = options["new_branch"] || false
+          log_message("Checking dataset", Thor::Shell::Color::BLUE) unless options[:sync]
+          local_idx = @dataset.generate_idx
+          result = @dataset.compare_idx(new_branch, commit=@dataset.last_local_commit, local_idx= local_idx, force=force, next_commit= next_commit)
+
+        commit_sha1 = result["result"]["commit"]
+          if commit_sha1 != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+            @dataset.remove_next_commit()
+            @files.delete_commit(next_commit)
+            log_message("Remote server has an updated version, please run `cnvrg data download` first or cnvrg data upload --new_branch", Thor::Shell::Color::BLUE)
+            exit(1)
+          end
+        check = Helpers.checkmark()
+
+
+        commit_time = result["result"]["commit_time"]
+        already_uploaded = result["result"]["uploaded"]
+        dirs_uploaded = []
+        files_uploaded = []
+        files_uploaded_names = []
+        if !already_uploaded.nil? and !already_uploaded.empty?
+
+          dirs_uploaded = already_uploaded["dirs"]
+          files_uploaded = already_uploaded["blobs"]
+          if !files_uploaded.nil? and !files_uploaded.empty?
+            files_uploaded_names = already_uploaded["blobs"].keys
+          end
+
+        end
+
+        result = result["result"]["tree"]
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          @files.delete_commit(next_commit)
+
+          log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, true)
+          return false
+        end
+        log_message("Creating new commit for dataset: #{@dataset.slug}", Thor::Shell::Color::BLUE) unless  options[:sync]
+
+
+        # upload / update
+        begin
+
+          parallel_options = {
+              :progress => {
+                  :title => "Upload Progress",
+                  :progress_mark => '=',
+                  :format => "%b>>%i| %p%% %t",
+                  :starting_at => 0,
+                  :total => (result["added"] + result["updated_on_local"]).size,
+                  :autofinish => true
+              },
+              in_threads: ParallelThreads,
+              isolation: true
+          }
+          successful_updates = []
+          update_count =0
+          update_total = (result["added"] + result["updated_on_local"] + result["deleted"]).size
+
+          if (result["added"] + result["updated_on_local"]).size > 0
+            begin
+              upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
+                absolute_path = "#{@dataset.local_path}/#{f}"
+                relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+                if File.directory?(absolute_path)
+                  if dirs_uploaded.include? f
+                    #Already uploaded and didn't finish commit
+                    log_message("Already uploaded dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+
+                    next
+                  end
+                  log_message("uploading dir: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+
+                  resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+                  if resDir
+                    update_count += 1
+                    successful_updates<< relative_path
+                  else
+                    log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
+
+                    raise Parallel::Kill
+                  end
+
+                else
+                  if files_uploaded_names.include? f
+                    log_message("Already uploaded file: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+
+                    next
+                  end
+                  log_message("uploading: #{f}", Thor::Shell::Color::BLUE, options["verbose"])
+
+                  res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+                  if res
+                    update_count += 1
+
+                    successful_updates<< relative_path
+                  else
+                    log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
+
+                    raise Parallel::Kill
+
+                  end
+                end
+              end
+            rescue SignalException
+              log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+
+
+              exit(1)
+            end
+          end
+
+        end
+          successful_updates = upload_result.to_a + dirs_uploaded + files_uploaded_names
+        successful_deletions =[]
+
+
+          # delete
+
+          deleted = update_deleted(result["deleted"])
+          begin
+
+            deleted_result = Parallel.map(deleted, in_threads: ParallelThreads) do |f|
+
+              relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+              if relative_path.end_with?("/")
+                log_message("deleting dir: #{f}", Thor::Shell::Color::RED, options["verbose"])
+
+                if @files.delete_dir(f, relative_path, commit_sha1)
+                  successful_deletions << f
+                else
+                  log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
+                  raise Parallel::Kill
+
+                end
+              else
+                log_message("deleteing file: #{f}", Thor::Shell::Color::RED, options["verbose"])
+
+                if @files.delete_file(f, relative_path, commit_sha1)
+                  successful_deletions << f
+
+                else
+                  log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
+                  raise Parallel::Kill
+
+
+                end
+              end
+            end
+
+
+          successful_deletions += successful_deletions.select { |x| not x.nil? }
+
+
+          successful_updates = successful_updates.select { |x| not x.nil? }
+
+          update_count = successful_updates.size
+
+        rescue SignalException
+          say "User aborted", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+          log_error(e)
+          exit(0)
+        end
+        if !result["deleted"].nil? and !result["deleted"].empty?
+          update_count += result["deleted"].size
+        end
+
+        if update_count == update_total
+          begin
+            @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
+
+            return true
+          rescue => e
+            log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+
+            exit(1)
+
+          end
+
+        end
+      rescue => e
+
+        log_message("Exception while trying to upload, look at the log for more details", Thor::Shell::Color::RED)
+        log_error(e)
+
+        exit(1)
+      rescue SignalException
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        exit(1)
+      end
+
+    end
+
+
+    desc 'upload', 'Upload updated files'
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
+    method_option :message, :type => :string, :aliases => ["-m", "--message"], :default => ""
+    method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
+
+    def upload(link = false, sync = false, direct = false, ignore_list = "")
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+
+        @project = Project.new(get_project_home)
+        commit_msg = options["message"]
+        if commit_msg.nil? or commit_msg.empty?
+          commit_msg = ""
+        end
+
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        ignore = options[:ignore] || ""
+        force = options[:force] || false
+
+        if ignore.nil? or ignore.empty?
+          ignore = ignore_list
+        end
+        data_ignore = data_dir_include()
+        if !data_ignore.nil?
+          if ignore.nil? or ignore.empty?
+            ignore = data_ignore
+          else
+            ignore = "#{ignore},#{data_ignore}"
+          end
+        end
+        if !@project.update_ignore_list(ignore)
+          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
+        end
+        new_branch = options["new_branch"] || false
+
+        if options["sync"] or sync
+          new_branch_exp = @project.get_new_branch
+          if new_branch_exp
+            new_branch = new_branch_exp
+          end
+        end
+
+        result = @project.compare_idx(new_branch, force: force, deploy: options["deploy"])
+        commit = result["result"]["commit"]
+        if !link
+          if ((commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?) and !force)
+
+            log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
+            exit(1)
+          end
+
+          log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
+        end
+        result = result["result"]["tree"]
+        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
+        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
+        #   if all.size == 1
+        #     num = "conflict"
+        #   else
+        #     num =  "conflicts"
+        #   end
+        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
+        #   say "#{all.join("\n")}"
+        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
+        #   exit(1)
+        #
+        # end
+        check = Helpers.checkmark()
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+        if options["verbose"]
+          if update_total == 1
+            log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
+          else
+            log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
+          end
+        else
+          log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
+
+        end
+        # Start commit
+        commit_sha1 = @files.start_commit(new_branch, force: force)["result"]["commit_sha1"]
+
+        # upload / update
+        begin
+
+          parallel_options = {
+              :progress => {
+                  :title => "Upload Progress",
+                  :progress_mark => '=',
+                  :format => "%b>>%i| %p%% %t",
+                  :starting_at => 0,
                   :total => (result["added"] + result["updated_on_local"]).size,
                   :autofinish => true
               },
@@ -1931,78 +2362,309 @@ module Cnvrg
           update_count += result["deleted"].size
         end
 
-        if update_count == update_total
-          res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
-          if (Cnvrg::CLI.is_response_success(res, false))
-            # save idx
-            begin
-              @project.update_idx_with_files_commits!((successful_deletions + successful_updates), res["result"]["commit_time"])
+        if update_count == update_total
+          res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
+          if (Cnvrg::CLI.is_response_success(res, false))
+            # save idx
+            begin
+              @project.update_idx_with_files_commits!((successful_deletions + successful_updates), res["result"]["commit_time"])
+
+              @project.update_idx_with_commit!(commit_sha1)
+            rescue => e
+              @files.rollback_commit(commit_sha1)
+              log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
+              log_error(e)
+              exit(1)
+
+            end
+            image = is_project_with_docker(Dir.pwd)
+            if image and image.is_docker
+              image.update_image_activity(commit_sha1, nil)
+            end
+
+            if options["verbose"]
+              log_message("#{check} Done", Thor::Shell::Color::BLUE)
+              if successful_updates.size > 0
+                successful_updates.flatten!
+                log_message("Updated:", Thor::Shell::Color::GREEN)
+                suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
+                log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
+              end
+              if successful_deletions.size > 0
+                successful_deletions.flatten!
+
+                log_message("Deleted:", Thor::Shell::Color::GREEN)
+                del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
+                log_message(del.join("\n"), Thor::Shell::Color::GREEN)
+              end
+              log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
+            else
+              if (options["sync"] or sync) and direct
+                log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
+
+              else
+                log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
+
+              end
+
+            end
+
+          else
+            @files.rollback_commit(commit_sha1)
+            log_message("Error: couldn't commit changes,  Rolling Back all changes.", Thor::Shell::Color::RED)
+          end
+        else
+          @files.rollback_commit(commit_sha1)
+          log_message("Error: uploaded only: #{update_count} / #{update_total}, \n Rolling back", Thor::Shell::Color::RED)
+
+        end
+      rescue => e
+
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+        log_error(e)
+        @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
+        puts e.message
+
+        exit(1)
+      rescue SignalException
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
+        exit(1)
+      end
+
+    end
+    desc 'download_file_data', 'Download one data files'
+    method_option :verbose, :type => :boolean, :aliases => ["-v", "--verbose"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :desc => "download specified commit", :default => nil
+    method_option :link, :type => :boolean, :aliases => ["-l","--link"], :desc => "download specified commit", :default => false
+    method_option :download_path, :type => :string, :aliases => ["-p", "--path"], :desc => "download specified commit", :default => ""
+    method_option :remote, :type => :boolean, :aliases => ["-r","--remote"], :desc => "remote download", :default => false
+    method_option :json, :type => :boolean, :aliases => ["-j","--json"],:default => false,   :desc => "response as json"
+
+    def download_file_data(file_path, *dataset_path)
+
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      begin
+        if dataset_path.nil? or dataset_path.empty?
+          dataset_path = Dir.pwd
+        elsif dataset_path.is_a? Array
+          dataset_path = dataset_path[0]
+        end
+        dataset_dir = is_cnvrg_dir(dataset_path)
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        commit_to_download = options["commit"] || nil
+        as_link  = options["link"] || false
+        download_path = options["path"]
+        remote = options["remote"]
+        as_json = options["json"]
+        if remote
+          download_path = "/data/#{@dataset.title}/"
+
+        else
+          download_path = dataset_dir
+        end
+        if !as_json
+
+          log_message("Downloading file", Thor::Shell::Color::BLUE)
+
+        end
+        res = @files.download_file_s3(file_path, file_path, download_path, conflict=false, commit_sha1=commit_to_download, as_link=as_link)
+        if as_link
+          puts res
+        else
+          if res
+            if as_json
+              response = {"status":"success"}
+              puts response.to_json
+            else
+              log_message("#{Helpers.checkmark()} File #{file_path} was successfully downloaded", Thor::Shell::Color::GREEN)
+
+            end
+          end
+
+
+        end
+
+      rescue =>e
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
+        log_error(e)
+        exit(1)
+      end
+    end
+
+    desc 'download_data_new', 'Download updated files'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => ""
+    method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => false
+
+      begin
+        def download_data_new(verbose=false,sync=false, commit=nil,all_files=true)
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        commit_to_download = commit
+        local_idx = @dataset.generate_idx
+        all_files = all_files
+        if !commit_to_download.nil? and !commit_to_download.empty?
+          res = @dataset.compare_idx_download( commit=commit_to_download, local_idx= local_idx,all_files=all_files)
+        else
+          res = @dataset.compare_idx_download( commit=nil, local_idx= local_idx,all_files=all_files)
+
+        end
+
+        result = res["result"]["tree"]
+
+        commit = res["result"]["commit"]
+        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
+          return true
+        end
+        update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
+
+
+        successful_changes = 0
+        if update_total ==1
+          log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !sync)
+        elsif update_total == 0
+          log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
+          return true
+        elsif options["verbose"]
+          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
+        else
+          log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
+
+        end
+        parallel_options = {
+            :progress => {
+                :title => "Download Progress",
+                :progress_mark => '=',
+                :format => "%b>>%i| %p%% %t",
+                :starting_at => 0,
+                :total => result["updated_on_server"].size,
+                :autofinish => true
+            },
+            in_threads: ParallelThreads
+        }
+        conflicts = 0
+        if !result["conflicts"].empty?
+          begin
+
+
+            conflicts_result = Parallel.map(result["conflicts"], in_threads: ParallelThreads) do |f|
+
+              relative_path = f.gsub(/^#{@dataset.local_path}/, "")
+              log_message("downloading: #{f}.conflict", Thor::Shell::Color::BLUE, verbose)
+
+              if @files.download_file_s3(f, relative_path, dataset_dir,  conflict=true, commit_sha1=commit)
+                conflicts +=1
+              else
+                log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
+                raise Parallel::Kill
+
+              end
+            end
+          rescue Interrupt
+
+            log_message("Failed to download files", Thor::Shell::Color::RED)
+
+            exit(1)
+          end
+        end
+
+
+        successful_changes += conflicts
+        downloads = 0
+        if !result["updated_on_server"].empty?
+
+            updated_on_server_result = Parallel.map(result["updated_on_server"], parallel_options) do |f|
+              begin
+              relative_path = f.gsub(/^#{@dataset.local_path}/, "")
+              if f.end_with? "/"
+                # dir
+                if !all_files
+                  log_message("downloading dir: #{f}", Thor::Shell::Color::BLUE, verbose)
+                  if @files.download_dir(dataset_dir, relative_path)
+                    downloads +=1
+                  else
+                    log_message("Couldn't create directory: #{f}", Thor::Shell::Color::RED)
+                    raise Parallel::Kill
+
 
-              @project.update_idx_with_commit!(commit_sha1)
-            rescue => e
-              @files.rollback_commit(commit_sha1)
-              log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
-              log_error(e)
-              exit(1)
+                  end
+                end
 
-            end
-            image = is_project_with_docker(Dir.pwd)
-            if image and image.is_docker
-              image.update_image_activity(commit_sha1, nil)
-            end
+              else
+                # blob
+                log_message("downloading file: #{f}", Thor::Shell::Color::BLUE, verbose)
 
-            if options["verbose"]
-              log_message("#{check} Done", Thor::Shell::Color::BLUE)
-              if successful_updates.size > 0
-                successful_updates.flatten!
-                log_message("Updated:", Thor::Shell::Color::GREEN)
-                suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-                log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
-              end
-              if successful_deletions.size > 0
-                successful_deletions.flatten!
+                if @files.download_file_s3(f, relative_path, dataset_dir,conflict=false, commit_sha1=commit)
+                  downloads +=1
 
-                log_message("Deleted:", Thor::Shell::Color::GREEN)
-                del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-                log_message(del.join("\n"), Thor::Shell::Color::GREEN)
+                else
+
+                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
+                  raise Parallel::Kill
+
+
+                end
               end
-              log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
-            else
-              if (options["sync"] or sync) and direct
-                log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
+              rescue Interrupt
+                log_message("Couldn't download", Thor::Shell::Color::RED)
 
-              else
-                log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
+                exit(1)
 
               end
 
             end
+            successful_changes += downloads
+
+        end
+
+        deleted = result["deleted"].to_a
+        delete_res = @files.delete_commit_files_local(deleted)
+        if !delete_res
+          log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
+          log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+          exit(1)
+
+        end
+        successful_changes += deleted.size
+
 
+
+        if update_total == successful_changes
+          # update idx with latest commit
+          @dataset.update_idx_with_commit!(commit)
+          check = Helpers.checkmark()
+
+          if options["verbose"]
+            log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
+            log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
+            log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
           else
-            @files.rollback_commit(commit_sha1)
-            log_message("Error: couldn't commit changes,  Rolling Back all changes.", Thor::Shell::Color::RED)
+            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
-        else
-          @files.rollback_commit(commit_sha1)
-          log_message("Error: uploaded only: #{update_count} / #{update_total}, \n Rolling back", Thor::Shell::Color::RED)
+          return true
 
         end
       rescue => e
 
-        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
         log_error(e)
-        @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
-        puts e.message
-
         exit(1)
       rescue SignalException
-
         say "\nAborting", Thor::Shell::Color::BLUE
-        say "\nRolling back all changes", Thor::Shell::Color::BLUE
-        @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
         exit(1)
       end
-
     end
 
     desc 'download', 'Download updated files'
@@ -2219,6 +2881,8 @@ module Cnvrg
     end
 
 
+
+
     desc 'jump', 'jump to specific commit'
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
 
@@ -2428,7 +3092,9 @@ module Cnvrg
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
     method_option :sync_before_terminate, :type => :boolean, :aliases => ["-sbt", "--sync_before_terminate"], :default => false
-    method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--perodic_sync"], :default => ""
+    method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => nil #//15,30,45,60
+    method_option :max_time, :type => :string, :aliases => [ "--max_time"], :default => nil
+    method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
 
 
     def run(*cmd)
@@ -2451,6 +3117,8 @@ module Cnvrg
       sync_before_terminate = options["sync_before_terminate"]
       periodic_sync = options["periodic_sync"]
       force = options["force"]
+      max_time = options["max_time"]
+      dataset_only_tree = options["dataset_only_tree"]
 
       options_hash = Hash[options]
       real_options = []
@@ -2464,8 +3132,7 @@ module Cnvrg
         return
       else
         if !periodic_sync.nil? and !periodic_sync.empty?
-          if /^\d{2}m$/ === periodic_sync
-            periodic_sync = periodic_sync.gsub("m","")
+          if /^\d{2}$/ === periodic_sync
             if !["15","30","45","60"].include? periodic_sync
               log_message("periodic sync can only be every 15m, 30m, 45m or 60m", Thor::Shell::Color::RED)
               exit(1)
@@ -2488,7 +3155,8 @@ module Cnvrg
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
                :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
-               :periodic_sync => periodic_sync
+                :max_time => max_time,
+               :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree
         return
       end
 
@@ -2523,7 +3191,6 @@ module Cnvrg
     method_option :sync_before_terminate, :type => :boolean, :aliases => ["-sbt", "--sync_before_terminate"], :default => false
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => ""
     def exec(*cmd)
-
       log = []
       cpu_average = 0
       memory_average = 0
@@ -2834,190 +3501,6 @@ module Cnvrg
         exit(1)
       end
     end
-
-    desc '', '', :hide => true
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
-    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
-
-    def exec_docker(*cmd)
-      log = []
-      cpu_average = 0
-      memory_average = 0
-      verify_logged_in()
-      log_start(__method__, args, options)
-      project_home = "/home/ds/notebooks"
-      @project = Project.new(project_home)
-      sync_before = options["sync_before"]
-      sync_after = options["sync_after"]
-      print_log = options["log"]
-      title = options["title"]
-      email_notification = options["email_notification"]
-      upload_output = options["upload_output"]
-      time_to_upload = calc_output_time(upload_output)
-      @image = is_project_with_docker(project_home)
-
-
-      begin
-        start_commit = @project.last_local_commit
-        cmd = cmd.join("\s")
-
-        log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
-
-        @exp = Experiment.new(@project.owner, @project.slug)
-        machine_activity = @exp.get_machine_activity(project_home)
-
-        platform = RUBY_PLATFORM
-        machine_name = Socket.gethostname
-        begin
-
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
-          unless @exp.slug.nil?
-            real = Time.now
-            exp_success = true
-            memory_total = []
-            cpu_total = []
-            start_loop = Time.now
-            begin
-              PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
-                begin
-                  stdout.each do |line|
-                    cur_time = Time.now
-                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
-                    monitor_by = monitor.split(" ")
-                    memory = monitor_by[2]
-                    cpu = monitor_by[1]
-                    memory_total << memory.to_f
-                    cpu_total << cpu.to_f
-                    real_time = Time.now - real
-
-                    cur_log = {time: cur_time,
-                               message: line,
-                               type: "stdout",
-                               real: real_time}
-                    if print_log
-                      puts cur_log
-                    end
-                    log << cur_log
-
-                    begin
-                      if time_to_upload != 0
-                        if time_to_upload <= Time.now - start_loop
-                          #upload current log
-                          cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
-                          memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
-
-                          @exp.upload_temp_log(log, cpu_average, memory_average)
-                          log = []
-                          start_loop = Time.now
-                        end
-
-                      end
-                    rescue
-                      log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
-                    end
-
-                  end
-
-
-                  if stderr
-
-                    stderr.each do |err|
-
-                      log << {time: Time.now, message: err, type: "stderr"}
-                    end
-                  end
-
-                rescue Errno::EIO => e
-                  break
-                rescue Errno::ENOENT
-
-                  exp_success = false
-
-                  log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
-                rescue PTY::ChildExited
-                  exp_success = false
-                  log_message("The process exited!", Thor::Shell::Color::RED)
-                rescue => e
-                  log_message("Error occurred, aborting", Thor::Shell::Color::RED)
-                  log_error(e)
-                  exit(0)
-                end
-                ::Process.wait pid
-                cpu_average = cpu_total.inject(0) {|sum, el| sum + el}.to_f / cpu_total.size
-                memory_average = memory_total.inject(0) {|sum, el| sum + el}.to_f / memory_total.size
-                exit_status = $?.exitstatus
-                if $?.exitstatus != 0
-                  exp_success = false
-                end
-                if !exp_success
-                  if !Cnvrg::Helpers.internet_connection?
-                    wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
-                    if wait_offline
-                      log_message("Waiting until your'e online..", Thor::Shell::Color::BLUE)
-                      while !Cnvrg::Helpers.internet_connection?
-                      end
-                      exit_status = 0
-                    else
-                      log_message("Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED)
-                      exit(0)
-                    end
-                  else
-
-                    end_commit = @project.last_local_commit
-                    res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-                    @image.update_image_activity(@project.last_local_commit, @exp.slug)
-                    log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
-                    exit(0)
-                  end
-
-                end
-                if sync_after
-                  log_message("Syncing project after running", Thor::Shell::Color::BLUE)
-                  # Sync after run
-                  download()
-                  upload()
-                  log_message("Done Syncing", Thor::Shell::Color::BLUE)
-                end
-                end_commit = @project.last_local_commit
-
-                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-                @image.update_image_activity(@project.last_local_commit, @exp.slug)
-
-                check = Helpers.checkmark()
-                log_message("#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
-              end
-            rescue => e
-              log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
-              log_error(e)
-              exit(1)
-            end
-
-
-          end
-
-        end
-      rescue => e
-        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
-        log_error(e)
-
-      rescue SignalException
-        exit_status = -1
-        end_commit = @project.last_local_commit
-        if !@exp.nil?
-
-          res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-        end
-
-        say "\nAborting"
-
-        exit(1)
-      end
-    end
-
     desc '', '', :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
@@ -3034,6 +3517,10 @@ module Cnvrg
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
+    method_option :max_time, :type => :string, :aliases => [ "--max_time"], :default => nil
+    method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
+    method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => nil
+    method_option :sync_before_terminate, :type => :boolean, :aliases => ["-sbt", "--sync_before_terminate"], :default => false
 
     def exec_remote(*cmd)
       verify_logged_in(true)
@@ -3047,6 +3534,21 @@ module Cnvrg
         data_commit = options["data_commit"] || nil
         sync_before = options["sync_before"]
         force = options["force"]
+        max_time = options["max_time"]
+        if !max_time.nil? and !max_time.empty?
+          max_time = max_time.to_i
+          if max_time <=0
+            log_message("Max time for experiment should be more than 0 minutess", Thor::Shell::Color::RED)
+            exit(1)
+          end
+        end
+        periodic_sync = options["periodic_sync"]
+        sync_before_terminate = options["sync_before_terminate"]
+        dataset_only_tree = options["dataset_only_tree"]
+        ds_sync_options = 0
+        if dataset_only_tree
+          ds_sync_options = 1
+        end
 
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
@@ -3081,7 +3583,7 @@ module Cnvrg
         end
 
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3105,13 +3607,6 @@ module Cnvrg
           if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
             image_slug = "cnvrg_gpu"
           end
-          # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
-          # if default
-          #   image = Images.new(working_dir, default_image_name)
-          #   image_slug = image.image_slug
-          # else
-          #   exit(0)
-          # end
         else
           image_slug = image.image_slug
         end
@@ -3135,7 +3630,8 @@ module Cnvrg
         end
         log_message("Running remote experiment", Thor::Shell::Color::BLUE)
         exp = Experiment.new(project.owner, project.slug)
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
+                              periodic_sync, sync_before_terminate, max_time, ds_sync_options)
         if Cnvrg::CLI.is_response_success(res)
           # if res["result"]["machine"] == -1
           #   say "There are no available machines", Thor::Shell::Color::BLUE
@@ -3194,7 +3690,6 @@ module Cnvrg
         exit(1)
       end
     end
-
     desc 'deploy', 'Deploys model to production'
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
     method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
@@ -3323,6 +3818,7 @@ module Cnvrg
     method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
 
 
     desc 'notebook', 'starts a notebook session remotely or locally'
@@ -3334,6 +3830,7 @@ module Cnvrg
       image = options["image"]
       data = options["data"]
       data_commit = options["data_commit"]
+      dataset_only_tree = options["dataset_only_tree"]
       if local
         invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
         return
@@ -3343,7 +3840,7 @@ module Cnvrg
         instance_type = get_instance_type(instances)
 
         invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
-               :data => data, :data_commit => data_commit
+               :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree
         return
 
       end
@@ -3462,6 +3959,7 @@ module Cnvrg
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :commit, :type => :string, :aliases => ["--commit"], :default => ""
+    method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
 
 
     def remote_notebook()
@@ -3474,7 +3972,11 @@ module Cnvrg
       data_commit = options["data_commit"]
       commit = options["commit"]
       notebook_type = options["notebook_type"]
-
+      dataset_only_tree = options["dataset_only_tree"]
+      ds_sync_options = 0
+      if dataset_only_tree
+        ds_sync_options = 1
+      end
 
       begin
         project = Project.new(working_dir)
@@ -3485,7 +3987,7 @@ module Cnvrg
         end
         invoke :sync, [false], []
         slug = ""
-        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type)
+        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options)
         if Cnvrg::CLI.is_response_success(res)
           slug = res["result"]["notebook_url"]
           log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)
@@ -5059,7 +5561,6 @@ module Cnvrg
 
       def set_owner(owner, username, url)
         home_dir = File.expand_path('~')
-
         begin
           if !File.directory? home_dir + "/.cnvrg"
             FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp"])
@@ -5186,12 +5687,12 @@ module Cnvrg
 
       def self.is_response_success(response, should_exit = true)
         begin
-          if response.nil?
-            if !Cnvrg::Helpers.internet_connection?
-              say("<%= color('Error:You seems to be offline', RED) %>")
-              $LOG.error message: "offline connection", type: "error"
-
-            end
+          if response.nil? or !response
+            # if !Cnvrg::Helpers.internet_connection?
+            #   say("<%= color('Error:You seems to be offline', RED) %>")
+            #   $LOG.error message: "offline connection", type: "error"
+            #
+            # end
             if should_exit
               exit(1)
             else
@@ -5214,12 +5715,15 @@ module Cnvrg
             end
           end
           return true
-        rescue => e
-          puts e.message
-          puts e.backtrace
+          rescue => e
+            puts e.message
+            puts e.backtrace
+          rescue SignalException
+            say "Aborting"
+          end
+
         end
 
-      end
 
       def self.get_owner
         home_dir = File.expand_path('~')
@@ -5341,27 +5845,27 @@ module Cnvrg
 
       end
 
+        def verify_logged_in(in_dir=true)
+          begin
+          log_handler()
+          auth = Cnvrg::Auth.new
+          unless auth.is_logged_in?
+            say 'You\'re not logged in', Thor::Shell::Color::RED
+            say 'Please log in via `cnvrg login`', Thor::Shell::Color::BLUE
+            exit(1)
+          end
 
-      def verify_logged_in(in_dir = true)
-        log_handler()
-        auth = Cnvrg::Auth.new
-        unless auth.is_logged_in?
-          say 'You\'re not logged in', Thor::Shell::Color::RED
-          say 'Please log in via `cnvrg login`', Thor::Shell::Color::BLUE
-          exit(1)
-        end
-
-        # if !Helpers.internet_connection?
-        #   wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
-        #   if wait_offline
-        #     say "Waiting until your'e online..", Thor::Shell::Color::BLUE
-        #     while !Cnvrg::Helpers.internet_connection?
-        #     end
-        #   else
-        #     say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
-        #     exit(0)
-        #   end
-        # end
+          # if !Helpers.internet_connection?
+          #   wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+          #   if wait_offline
+          #     say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+          #     while !Cnvrg::Helpers.internet_connection?
+          #     end
+          #   else
+          #     say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
+          #     exit(0)
+          #   end
+          # end
 
         config = YAML.load_file(File.expand_path('~') + "/.cnvrg/config.yml")
         version_date = config.to_h[:version_last_check]
@@ -5385,8 +5889,10 @@ module Cnvrg
         #verify tmp dirs exist
         home_dir = File.expand_path('~')
 
-        FileUtils.mkdir_p([home_dir + "/.cnvrg", home_dir + "/.cnvrg/tmp", home_dir + "/.cnvrg/tmp_files"])
+          FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp", home_dir+"/.cnvrg/tmp_files"])
+          rescue SignalException
 
+          end
 
       end
 
@@ -5746,29 +6252,10 @@ module Cnvrg
 
       end
 
-      def with_progress_bar
-        not_done = true
-        upload_progress_bar = ProgressBar.create(:title => "Upload progress",
-                                                 :format => '%a <%B> %p%% %t',
-                                                 :starting_at => 0,
-                                                 :total => 600)
-        pb = Thread.new do
-          while not_done do
-            upload_progress_bar.increment
-            sleep(1)
-
-          end
-        end
-
-        yield.tap do # After yielding to the block, save the return value
-          not_done = false # Tell the thread to exit, cleaning up after itself…
-          pb.join # …and wait for it to do so.
-        end
-      end
     end
+  end
 
 
-  end
 end