cnvrg 0.0.1530000 → 0.0.1540000

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dfd7e3915a7ca0c235ac844cbb19de4ecd985ff8
-  data.tar.gz: e5435e7a078993757e7e0bcc937632cd73d16ffd
+  metadata.gz: 021cf92b7a10351f3e2b5dffb061acbba7e335e2
+  data.tar.gz: 087dac2c6aacf82f7ca15c9e51f4858e7cfd1035
 SHA512:
-  metadata.gz: 05388102f35ca1434e624cdab9bdad58c52e28594fcc9b5a18edb9ff7a2953341fbcf9e141a1bd751fcfb9136763afba99a0f3adfe841600c84375d674dd7580
-  data.tar.gz: 9350e72abedd72550692d0b4fa65f9f6c95ad5f56d9f343480ce3c360a02b8ee369ba9fc9dac37dda8837f1c581893c3d2755bc37a46c3ddbf0dae23e4cad349
+  metadata.gz: 34b286d7d0c4e9c3d93acc09c3f857674d0b75f575730bfa7bae48b8fa5396cf631714b53bb7c99b48d5e7603925574011cef44801e235e208b2992fcf9ef2bb
+  data.tar.gz: 44dacf68e883b199b0ceabc6921a5c1d853155942d8ad5dadb207effeb4e7096ae54e61edf819ca2385a5142b1b49a274648da7ab259faac5da721d4dd154e1c

data/cnvrg.gemspec

@@ -34,6 +34,8 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'aws-sdk'
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
+    spec.add_runtime_dependency 'parallel', '~> 1.12.0'
+    
     spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
     spec.add_runtime_dependency 'launchy', '~> 2.4'
     spec.add_runtime_dependency 'docker-api', '~> 1.33'

data/lib/cnvrg/api.rb

@@ -81,14 +81,17 @@ module Cnvrg
 
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
-                file_base = File.basename(data[:relative_path])
-                temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
-                FileUtils.touch(temp_path)
+                  file_base = File.basename(data[:relative_path])
+                    temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
+                    FileUtils.touch(temp_path)
+
+
                 data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
 
                 response = conn.post "#{endpoint_uri}/#{resource}", data
 
-                FileUtils.rm(temp_path)
+                FileUtils.rm_rf(temp_path)
+
 
                 if parse_request == true
                     JSON.parse(response.body)

data/lib/cnvrg/cli.rb

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 require "pty"
+require 'etc'
+require 'parallel'
 require "open4"
 require 'netrc'
 require 'net/http'
@@ -22,7 +24,6 @@ require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
 require 'cnvrg/ssh'
-require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
 require 'docker'
@@ -35,6 +36,7 @@ require 'active_support/all'
 require 'thor'
 require 'pathname'
 require 'enumerator'
+require 'ruby-progressbar'
 
 class Thor
   module Base
@@ -143,6 +145,10 @@ module Cnvrg
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
     IP="localhost"
     PORT=7654
+
+    ParallelThreads ||= 10
+    ParallelProcesses ||= Parallel.processor_count
+
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
       def is_thor_reserved_word?(word, type)
@@ -203,11 +209,11 @@ module Cnvrg
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
         owner = config.to_h[:owner]
-        compression_path =  "#{File.expand_path('~')}/.cnvrg/tmp"
+        compression_path = "#{File.expand_path('~')}/.cnvrg/tmp"
 
         say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
         if config.empty?
-          config = {owner: "", username: "", version_last_check: get_start_day(), api: url,compression_path: compression_path }
+          config = {owner: "", username: "", version_last_check: get_start_day(), api: url, compression_path: compression_path}
         else
           if !config.to_h[:compression_path].nil?
             compression_path = config.to_h[:compression_path]
@@ -215,7 +221,7 @@ module Cnvrg
           config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
         end
         res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
-        if Cnvrg::CLI.is_response_success(res,false)
+        if Cnvrg::CLI.is_response_success(res, false)
 
           checks = Helpers.checkmark
 
@@ -249,8 +255,8 @@ module Cnvrg
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
 
-        compression_path =  "#{home_dir}/.cnvrg/tmp"
-        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url,compression_path:compression_path}
+        compression_path = "#{home_dir}/.cnvrg/tmp"
+        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url, compression_path: compression_path}
         File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
         say "Done"
       rescue
@@ -326,12 +332,12 @@ module Cnvrg
     end
 
     desc 'set_compression_path', 'set compression path'
-    method_option :reset, :type => :boolean, :aliases => ["-r","--reset"], :default => false
+    method_option :reset, :type => :boolean, :aliases => ["-r", "--reset"], :default => false
 
     def set_compression_path(*compression_path)
       begin
         if (compression_path.nil? or compression_path.empty?) and options["reset"]
-          compression_path =  ["#{File.expand_path('~')}/.cnvrg/tmp"]
+          compression_path = ["#{File.expand_path('~')}/.cnvrg/tmp"]
         end
         compression_path = compression_path.join(" ")
         if !Dir.exist? compression_path
@@ -348,7 +354,7 @@ module Cnvrg
         end
         config = YAML.load_file(path)
         config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
-                  version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
+                      version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
         File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config_new.to_yaml }
         checks = Helpers.checkmark
         say "#{checks} Done", Thor::Shell::Color::GREEN
@@ -360,11 +366,13 @@ module Cnvrg
     end
 
 
-
     desc 'login', 'Authenticate with cnvrg.io platform'
 
     def login
       begin
+        log_handler()
+        log_start(__method__, args, options)
+
         cmd = HighLine.new
 
         say 'Authenticating with cnvrg', Thor::Shell::Color::YELLOW
@@ -374,7 +382,7 @@ module Cnvrg
         @email, token = netrc[Cnvrg::Helpers.netrc_domain]
 
         if @email and token
-          say 'Seems you\'re already logged in', Thor::Shell::Color::BLUE
+          log_message('Seems you\'re already logged in', Thor::Shell::Color::BLUE)
           exit(0)
         end
         @email = ask("Enter your email:")
@@ -385,7 +393,8 @@ module Cnvrg
           netrc[Cnvrg::Helpers.netrc_domain] = @email, result["token"]
           netrc.save
 
-          say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
+          log_message("Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN)
+
           owners = result["owners"]
           urls = result["urls"]
           choose_owner = result["username"]
@@ -431,22 +440,24 @@ module Cnvrg
           # end
 
           if set_owner(choose_owner, result["username"], urls[ow_index])
-            say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
+            log_message("Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN)
 
           else
-            say "Setting default owenr has failed, logging out", Thor::Shell::Color::RED
+            log_message("Setting default owenr has failed, logging out", Thor::Shell::Color::RED)
 
             return logout()
           end
 
         else
-          say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
+          log_message("Failed to authenticate, wrong email/password", Thor::Shell::Color::RED)
 
           exit(1)
         end
-      rescue
+      rescue => e
+
+        log_message("Error Occurred, aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
-        say "Error Occurred, aborting", Thor::Shell::Color::RED
         logout()
         exit(1)
       rescue SignalException
@@ -462,10 +473,12 @@ module Cnvrg
 
     def logout
       begin
+        log_handler()
+        log_start(__method__, args, options)
         netrc = Netrc.read
         netrc.delete(Cnvrg::Helpers.netrc_domain)
         netrc.save
-        say "Logged out successfully.\n", Thor::Shell::Color::GREEN
+        log_message("Logged out successfully.\n", Thor::Shell::Color::GREEN)
       rescue SignalException
         say "\nAborting"
         exit(1)
@@ -482,16 +495,13 @@ module Cnvrg
         log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
         if (email = auth.get_email)
-          say "Logged in as: #{email}", Thor::Shell::Color::GREEN
+          log_message("Logged in as: #{email}", Thor::Shell::Color::GREEN)
         else
-          say "You're not logged in.", Thor::Shell::Color::RED
+          log_message("You're not logged in.", Thor::Shell::Color::RED)
         end
 
-        log_end(0)
 
       rescue SignalException
-        log_end(-1)
-
         say "\nAborting"
         exit(1)
       end
@@ -513,10 +523,9 @@ module Cnvrg
         if !docker_image.nil? and !docker_image.empty?
           docker = true
         end
-        say "Creating #{project_name}", Thor::Shell::Color::BLUE
+        log_message("Creating #{project_name}", Thor::Shell::Color::BLUE)
         if Dir.exists? project_name or File.exists? project_name
-          say "Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
-          log_end(1, "conflict with dir/file #{project_name}")
+          log_message("Conflict with dir/file #{project_name}", Thor::Shell::Color::RED)
           exit(1)
 
         end
@@ -532,24 +541,24 @@ module Cnvrg
               if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
                 image = pull(docker_image)
                 if image
-                  say "downloaded image: #{docker_image}"
+                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
                   @image = Images.new(working_dir, docker_image)
                 else
-                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
                   @project.revert(working_dir)
                   exit(1)
                 end
               else
-                say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
                 @project.revert(working_dir)
                 exit(1)
 
               end
             elsif docker_image_local.size == 1
-              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
               @image = Images.new(working_dir, docker_image_local[0])
             elsif docker_image_local.size >1
-              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
               image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
               image_name = image_name.strip
               @image = Images.new(working_dir, image_name)
@@ -558,18 +567,17 @@ module Cnvrg
 
           end
         else
-          say "Error creating project, please contact support.", Thor::Shell::Color::RED
+          log_message("Error creating project, please contact support.", Thor::Shell::Color::RED)
           @project.revert(working_dir)
-          log_end(1, "can't create project #{project_name}")
 
           exit(0)
         end
 
-        say "created project successfully", Thor::Shell::Color::GREEN
-        say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
+        log_message("created project successfully", Thor::Shell::Color::GREEN)
+        log_message("Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN)
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurred, aborting", Thor::Shell::Color::RED
+        log_message("Error occurred, aborting", Thor::Shell::Color::RED)
+        log_error(e)
         if Dir.exist? working_dir
 
           @project.revert(working_dir)
@@ -642,11 +650,10 @@ module Cnvrg
 
         sync = options["sync"]
         project_name =File.basename(Dir.getwd)
-        say "Linking #{project_name}", Thor::Shell::Color::BLUE
+        log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
         if File.directory?(Dir.getwd+"/.cnvrg")
           config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
-          say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
-
+          log_message("Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED)
           exit(0)
         end
         working_dir = Dir.getwd
@@ -662,47 +669,46 @@ module Cnvrg
               if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
                 image = pull(docker_image)
                 if image
-                  say "downloaded image: #{docker_image}"
+                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
                   @image = Images.new(working_dir, docker_image)
                 else
-                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
                   @project.revert(working_dir)
                   exit(1)
                 end
               else
-                say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
                 @project.revert(working_dir)
                 exit(1)
 
               end
             elsif docker_image_local.size == 1
-              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
               @image = Images.new(working_dir, docker_image_local[0])
             elsif docker_image_local.size >1
-              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
               image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
               @image = Images.new(working_dir, image_name)
             end
           end
           if sync
-            say "Syncing project", Thor::Shell::Color::BLUE
+            log_message("Syncing project", Thor::Shell::Color::BLUE)
             upload(true)
           end
 
           url = @project.url
           check = Helpers.checkmark
-          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
-          say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
+          log_message("#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)
 
         else
-          log_end(1, "can't link project")
           @project.revert(working_dir) unless @project.nil?
-          say "Error linking project, please contact support.", Thor::Shell::Color::RED
+          log_message("Error linking project, please contact support.", Thor::Shell::Color::RED)
           exit(0)
         end
+      rescue => e
+        log_error(e)
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting"
         exit(1)
@@ -719,11 +725,11 @@ module Cnvrg
         dataset_name =File.basename(Dir.getwd)
         if File.directory?(Dir.getwd+"/.cnvrg")
           config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
-          say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
+          log_message("Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED)
 
           exit(0)
         end
-        say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
+        log_message("Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE)
 
         working_dir = Dir.getwd
         owner = CLI.get_owner
@@ -733,19 +739,17 @@ module Cnvrg
 
           url = @dataset.url
           check = Helpers.checkmark
-          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
-          say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("#{check} Link finished successfully", Thor::Shell::Color::GREEN)
+          log_message("#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN)
 
         else
-          log_end(1, "can't create dataset")
           @dataset.revert(working_dir) unless @dataset.nil?
-          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
           exit(0)
         end
+      rescue => e
+        log_error(e)
       rescue SignalException
-        log_end(-1)
-
         say "\nAborting"
         exit(1)
       end
@@ -767,17 +771,15 @@ module Cnvrg
         Cnvrg::CLI.is_response_success(response)
         dataset_name = response["result"]["name"]
         if (Dir.exists? dataset_name)
-          say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
+          log_messgae("Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED)
           if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-            say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
-            log_end(1, "conflict with dir #{dataset_name}")
-
+            log_message("Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED)
             exit(1)
           end
 
         end
         if Dataset.clone(owner, dataset_name, slug)
-          say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
+          log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
 
           commit_to_clone = options["commit"] || nil
           working_dir = "#{Dir.pwd}/#{dataset_name}"
@@ -785,21 +787,23 @@ module Cnvrg
           @dataset.generate_idx()
 
 
-          download_data(false, false, path = working_dir, in_dir=false)
+          download_res = download_data(false, false, path = working_dir, in_dir=false)
+          if !download_res
+            exit(1)
+          end
 
 
           check = Helpers.checkmark
-          say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
 
         else
-          log_end(1, "can't create dataset")
           @dataset.revert(working_dir) unless @dataset.nil?
-          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          log_message("Error creating dataset, please contact support.", Thor::Shell::Color::RED)
           exit(0)
         end
+      rescue => e
+        log_error(e)
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting"
         exit(1)
@@ -826,7 +830,6 @@ module Cnvrg
         container.exec(command, tty: true)
 
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting"
         exit(1)
@@ -862,10 +865,8 @@ module Cnvrg
       if Dataset.init_container(owner, dataset_slug, dataset_name)
 
         say "init finished successfully", Thor::Shell::Color::GREEN
-        log_end(0)
 
       else
-        log_end(1, "can't create dataset")
         say "error creating dataset, please contact support.", Thor::Shell::Color::RED
         exit(0)
       end
@@ -894,33 +895,70 @@ module Cnvrg
 
         commit = res["commit"]
         if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
-          log_end(0)
+          log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync)) ? false : true)
           return true
         end
-        say "Downloading data", Thor::Shell::Color::BLUE
+        log_message("Downloading data", Thor::Shell::Color::BLUE)
 
         result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
 
         delete = result["result"]["delete"]
         commits = result["result"]["commits"]
         updated_idx = result["result"]["idx"]
-        commits.each do |c|
+        parallel_options = {
+            :progress => {
+                :title => "Download Progress",
+                :progress_mark => '=',
+                :format => "%b>>%i| %p%% %t",
+                :starting_at => 0,
+                :total => commits.size,
+                :autofinish => true
+            },
+            in_processes: ParallelProcesses,
+            in_thread: ParallelThreads
+        }
+
+        begin
+        tar_files = []
+        download_result = Parallel.map(commits, parallel_options) do |c|
+
           file_name = @files.download_data_file(c, dataset_dir)
 
           if file_name.eql? false or file_name.nil?
-            ##RETRY
-            next
+            count = 0
+            success_download = false
+            while count < 3 and !success_download
+              log_message("Couldn't download data files, retrying.. ", Thor::Shell::Color::BLUE)
+
+              file_name = @files.download_data_file(c, dataset_dir)
+              success_download = (file_name.eql? false or file_name.nil?)
+              count +=1
+
+            end
+            if count > 3 or !success_download
+              log_message("Couldn't download data files,revoking", Thor::Shell::Color::RED)
+
+              raise Parallel::Kill
+            end
           end
           file_path = "#{dataset_dir}/#{file_name}"
+          tar_files << file_path
           success = extarct_tar(file_path, dataset_dir)
           if !success
-            #TODO: ?
+            log_message("Couldn't extract data files,revoking", Thor::Shell::Color::RED)
+
+            raise Parallel::Kill
           end
 
           FileUtils.rm_rf([file_path])
 
         end
+        rescue Interrupt
+          @files.revoke_download(tar_files,updated_idx[:tree].keys )
+          return false
+        end
+
+
         to_delete = []
         delete.each do |d|
           to_delete << "#{dataset_dir}/#{d}"
@@ -930,28 +968,19 @@ module Cnvrg
         @dataset.update_idx(updated_idx)
 
 
-        # result["conflicts"].each do |f|
-        #   relative_path = f.gsub(/^#{@dataset.local_path}/, "")
-        #   if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
-        #     successful_changes << relative_path
-        #   end
-        #
-        # end
-        # update idx with latest commit
-        # @dataset.update_idx_with_commit!(commit)
         check = Helpers.checkmark()
-        say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
+        log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN)
+        return true
 
 
-        log_end(0)
       end
-    rescue
-      log_end(-1)
+    rescue => e
+      log_message("Error occurd, \nAborting", Thor::Shell::Color::BLUE)
+      log_error(e)
+      @files.revoke_download(tar_files,updated_idx[:tree].keys )
 
-      say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
       exit(1)
     rescue SignalException
-      log_end(-1)
       say "\nAborting", Thor::Shell::Color::BLUE
       exit(1)
     end
@@ -974,19 +1003,18 @@ module Cnvrg
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         ignore = options[:ignore] || []
         if !@dataset.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
         result = @dataset.compare_idx(false)
 
         commit = result["result"]["commit"]
         if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-          log_end(0)
 
-          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+          log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
           exit(1)
         end
 
-        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
+        log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, options["verbose"] ? true : false)
         result = result["result"]["tree"]
         # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
         #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
@@ -1004,8 +1032,7 @@ module Cnvrg
         check = Helpers.checkmark()
 
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_end(0)
-          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
+          log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
           return true
         end
         update_count = 0
@@ -1014,12 +1041,12 @@ module Cnvrg
         successful_deletions = []
         if options["verbose"]
           if update_total == 1
-            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
+            log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
           else
-            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
+            log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
           end
         else
-          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
+          log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
 
         end
 
@@ -1045,8 +1072,7 @@ module Cnvrg
                 successful_updates<< relative_path
               else
                 @files.rollback_commit(commit_sha1)
-                log_end(1, "can't upload, Rolling Back all changes")
-                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+                log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
                 exit(0)
               end
             end
@@ -1070,14 +1096,13 @@ module Cnvrg
           end
 
         rescue SignalException
-          log_end(-1)
           @files.rollback_commit(commit_sha1)
           say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
           exit(0)
         rescue => e
-          log_end(-1, e.message)
+          log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
+          log_error(e)
           @files.rollback_commit(commit_sha1)
-          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
           exit(0)
         end
         if !result["deleted"].nil? and !result["deleted"].empty?
@@ -1092,57 +1117,52 @@ module Cnvrg
 
               @dataset.update_idx_with_commit!(commit_sha1)
             rescue => e
-              log_end(-1, e.message)
+              log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
+              log_error(e)
               @files.rollback_commit(commit_sha1)
-              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
               exit(1)
 
             end
             if options["verbose"]
-              say "#{check} Done", Thor::Shell::Color::BLUE
+              log_message("#{check} Done", Thor::Shell::Color::BLUE)
               if successful_updates.size >0
-                say "Updated:", Thor::Shell::Color::GREEN
+                log_message("Updated:", Thor::Shell::Color::GREEN)
                 suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say suc.join("\n"), Thor::Shell::Color::GREEN
+                log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
               end
               if successful_deletions.size >0
-                say "Deleted:", Thor::Shell::Color::GREEN
+                log_message("Deleted:", Thor::Shell::Color::GREEN)
                 del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say del.join("\n"), Thor::Shell::Color::GREEN
+                log_message(del.join("\n"), Thor::Shell::Color::GREEN)
               end
-              say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+              log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
             else
               if (options["sync"] or sync) and direct
-                say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
+                log_message("#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN)
 
               else
-                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+                log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
 
               end
 
             end
 
-            log_end(0)
           else
             @files.rollback_commit(commit_sha1)
-            log_end(1, "error. Rolling Back all changes")
-            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
+            log_message("Error: Couldn't commit. \nRolling Back all changes.", Thor::Shell::Color::RED)
           end
         else
-          log_end(1, "error. Rolling Back all changes")
-          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
+          log_message("Error: Uploaded only #{update_count}/#{update_total} files, \nRolling back", Thor::Shell::Color::RED)
 
           @files.rollback_commit(commit_sha1)
         end
       rescue => e
+        log_message("Error occurd, \nAborting", Thor::Shell::Color::RED)
+        log_error(e)
 
-        log_end(-1)
-
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
         exit(1)
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting", Thor::Shell::Color::BLUE
         say "\nRolling back all changes", Thor::Shell::Color::BLUE
@@ -1159,7 +1179,7 @@ module Cnvrg
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
 
-    def upload_data_tar(ignore, verbose, sync,no_compression)
+    def upload_data_tar(ignore, verbose, sync, no_compression)
 
       begin
         verify_logged_in(true)
@@ -1170,29 +1190,27 @@ module Cnvrg
 
         @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
         if !@dataset.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED
+          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED)
           exit(1)
         end
-        say "Checking dataset", Thor::Shell::Color::BLUE
+        log_message("Checking dataset", Thor::Shell::Color::BLUE)
         local_idx = @dataset.generate_idx
         result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
 
 
         commit = result["result"]["commit"]
         if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-          log_end(0)
 
-          say "Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::YELLOW
+          log_message("Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::BLUE)
           exit(1)
         end
 
-        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
+        log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, verbose)
         result = result["result"]["tree"]
         check = Helpers.checkmark()
 
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_end(0)
-          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
+          log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN, (sync ? false : true))
           return true
         end
         update_count = 0
@@ -1221,7 +1239,7 @@ module Cnvrg
           @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
 
           @dataset.update_idx_with_commit!(commit_sha1)
-          say "Compressing data", Thor::Shell::Color::BLUE
+          log_message("Compressing data", Thor::Shell::Color::BLUE)
 
           home_dir = File.expand_path('~')
           compression_path = get_compression_path
@@ -1229,22 +1247,22 @@ module Cnvrg
           tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
           tar_files = (result["added"] + result["updated_on_local"]).join("\n")
           File.open(tar_files_path, 'w') { |f| f.write tar_files }
-          is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
+          is_tar = create_tar(dataset_dir, tar_path, tar_files_path, no_compression)
           if !is_tar
-            say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
+            log_message("ERROR: Couldn't compress data", Thor::Shell::Color::RED)
             FileUtils.rm_rf([tar_path]) if File.exist? tar_path
             FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
             @files.rollback_commit(commit_sha1)
-            say "Rolling Back all changes.", Thor::Shell::Color::RED
+            log_message("Rolling Back all changes.", Thor::Shell::Color::RED)
             exit(1)
           end
-          say "Uploading data", Thor::Shell::Color::BLUE
+          log_message("Uploading data", Thor::Shell::Color::BLUE)
           log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(tar_path)}.log"
           res = false
           res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
 
           if res
-            say "Commiting data", Thor::Shell::Color::BLUE
+            log_message("Commiting data", Thor::Shell::Color::BLUE)
 
             cur_idx = @dataset.get_idx.to_h
 
@@ -1255,13 +1273,13 @@ module Cnvrg
 
 
               @files.rollback_commit(commit_sha1)
-              say "Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED
+              log_message("Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED)
               exit(1)
             end
 
           else
             if File.exist? log_file
-             @files.upload_data_log_file(log_file,log_file,commit_sha1)
+              @files.upload_data_log_file(log_file, log_file, commit_sha1)
             end
 
 
@@ -1270,8 +1288,8 @@ module Cnvrg
 
 
             @files.rollback_commit(commit_sha1)
-            say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-            say "Error log: #{log_file}", Thor::Shell::Color::RED
+            log_message("Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
+            log_message("Upload error log: #{log_file}", Thor::Shell::Color::RED)
 
             exit(1)
           end
@@ -1281,11 +1299,10 @@ module Cnvrg
           FileUtils.rm_rf([tar_path, tar_files_path])
 
         rescue SignalException
-          log_end(-1)
           FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
           FileUtils.rm_rf([tar_path]) if File.exist? tar_path
           if File.exist? log_file
-            @files.upload_data_log_file(log_file,log_file,commit_sha1)
+            @files.upload_data_log_file(log_file, log_file, commit_sha1)
           end
 
 
@@ -1293,36 +1310,36 @@ module Cnvrg
           say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
           exit(0)
         rescue => e
-          puts e.message
+          log_error(e)
           if !Cnvrg::Helpers.internet_connection?
             say "Seems there is no internet connection", Thor::Shell::Color::RED
 
           end
           if File.exist? log_file
-            @files.upload_data_log_file(log_file,log_file,commit_sha1)
+            @files.upload_data_log_file(log_file, log_file, commit_sha1)
           end
           FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
           FileUtils.rm_rf([tar_path]) if File.exist? tar_path
 
-          log_end(-1, e.message)
           @files.rollback_commit(commit_sha1)
-          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
-          say "Error log: #{log_file}", Thor::Shell::Color::RED
+          log_message("Exception while trying to upload, \nRolling back,\n look at the log for more details", Thor::Shell::Color::RED)
+          log_message("Error log: #{log_file}", Thor::Shell::Color::RED)
 
 
           exit(0)
         end
-        say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
-
+        log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
 
 
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+
+        puts e.message
+        puts e.backtrace
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+        log_error(e)
         @files.rollback_commit(commit_sha1)
         exit(1)
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting", Thor::Shell::Color::BLUE
         say "\nRolling back all changes", Thor::Shell::Color::BLUE
@@ -1330,7 +1347,6 @@ module Cnvrg
         exit(1)
       end
 
-      log_end(0)
 
     end
 
@@ -1424,18 +1440,16 @@ module Cnvrg
         project_name = response["title"]
         commit_to_clone = options["commit"] || nil
 
-        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
+        log_message("Cloning #{project_name}", Thor::Shell::Color::BLUE)
         clone_resp = false
         if remote
           clone_resp = Project.clone_dir_remote(slug, owner, project_name)
           project_home = Dir.pwd
         else
           if (Dir.exists? project_name)
-            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
+            log_message("Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED)
             if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
-              log_end(1, "conflict with dir #{project_name}")
-
+              log_message("Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED)
               exit(1)
             end
 
@@ -1463,20 +1477,20 @@ module Cnvrg
               if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
                 image = pull(docker_image)
                 if image
-                  say "downloaded image: #{docker_image}"
+                  log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
                   @image = Images.new(working_dir, docker_image)
                 else
-                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
                   @project.revert(working_dir)
                   exit(1)
                 end
               end
 
             elsif docker_image_local.size == 1
-              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
               @image = Images.new(working_dir, docker_image_local[0])
             elsif docker_image_local.size >1
-              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
               image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
               @image = Images.new(working_dir, image_name)
             end
@@ -1484,35 +1498,65 @@ module Cnvrg
           end
 
           successful_changes = []
-          say "Downloading files", Thor::Shell::Color::BLUE
+          log_message("Downloading files", Thor::Shell::Color::BLUE)
           if !response["result"]["tree"].nil?
-            response["result"]["tree"].each do |f|
-              relative_path = f[0].gsub(/^#{@project.local_path}/, "")
-              if f[0].end_with? "/"
-                # dir
-                if @files.download_dir(f[0], relative_path, project_home)
-                  successful_changes << relative_path
-                end
-              else
-                # blob
+            parallel_options = {
+                :progress => {
+                    :title => "Download Progress",
+                    :progress_mark => '=',
+                    :format => "%b>>%i| %p%% %t",
+                    :starting_at => 0,
+                    :total => response["result"]["tree"].size,
+                    :autofinish => true
+                },
+                in_processes: ParallelProcesses,
+                in_thread: ParallelThreads
+            }
+            begin
+              clone_result = Parallel.map((response["result"]["tree"]), parallel_options) do |f|
 
-                if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
-                  successful_changes << relative_path
+                relative_path = f[0].gsub(/^#{@project.local_path}/, "")
+                if f[0].end_with? "/"
+                  # dir
+                  if @files.download_dir(f[0], relative_path, project_home)
+                    f
+                  else
+                    log_message("Could not create directory: #{f[0]}", Thor::Shell::Color::RED)
+                    raise Parallel::Kill
+                  end
+                else
+                  # blob
+
+                  if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
+                    f
+                    successful_changes << relative_path
+                  else
+                    log_message("Could not download file: #{f[0]}", Thor::Shell::Color::RED)
+                    raise Parallel::Kill
+
+                  end
                 end
               end
+            rescue Interrupt
+
+              log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+              @files.revoke_download([], response["result"]["tree"])
+              exit(1)
             end
+
+
           end
 
-          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
-          log_end(0)
+          successful_changes = clone_result
+
+          log_message("Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN)
         else
-          log_end(1, "can't create directory")
 
-          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
+          log_message("Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED)
           exit(1)
         end
       rescue SignalException
-        log_end(-1)
         say "\nAborting"
         exit(1)
       end
@@ -1534,49 +1578,46 @@ module Cnvrg
         result = @project.compare_idx(new_branch)["result"]
         commit = result["commit"]
         result = result["tree"]
-        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
+        log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE)
 
         if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("Project is up to date", Thor::Shell::Color::GREEN)
           return true
         end
         if result["added"].size > 0
-          say "Added files:\n", Thor::Shell::Color::BLUE
+          log_message("Added files:\n", Thor::Shell::Color::BLUE)
           result["added"].each do |a|
-            say "\t\tA:\t#{a}", Thor::Shell::Color::GREEN
+            log_message("\t\tA:\t#{a}", Thor::Shell::Color::GREEN)
           end
         end
 
         if result["deleted"].size > 0
-          say "Deleted files:\n", Thor::Shell::Color::BLUE
+          log_message("Deleted files:\n", Thor::Shell::Color::BLUE)
           result["deleted"].each do |a|
-            say "\t\tD:\t#{a}", Thor::Shell::Color::GREEN
+            log_message("\t\tD:\t#{a}", Thor::Shell::Color::GREEN)
           end
         end
         if result["updated_on_local"].size > 0
-          say "Local changes:\n", Thor::Shell::Color::BLUE
+          log_message("Local changes:\n", Thor::Shell::Color::BLUE)
           result["updated_on_local"].each do |a|
-            say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
+            log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
           end
         end
 
         if result["updated_on_server"].size > 0
-          say "Remote changes:\n", Thor::Shell::Color::BLUE
+          log_message("Remote changes:\n", Thor::Shell::Color::BLUE)
           result["updated_on_server"].each do |a|
-            say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
+            log_message("\t\tM:\t#{a}", Thor::Shell::Color::GREEN)
           end
         end
 
         if result["conflicts"].size > 0
-          say "Conflicted changes:\n", Thor::Shell::Color::BLUE
+          log_message("Conflicted changes:\n", Thor::Shell::Color::BLUE)
           result["conflicts"].each do |a|
-            say "\t\tC:\t#{a}", Thor::Shell::Color::RED
+            log_message("\t\tC:\t#{a}", Thor::Shell::Color::RED)
           end
         end
-        log_end(0)
       rescue SignalException
-        log_end(-1)
         say "\nAborting"
         exit(1)
       end
@@ -1634,7 +1675,7 @@ module Cnvrg
           end
         end
         if !@project.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
         new_branch = options["new_branch"] || false
 
@@ -1648,13 +1689,12 @@ module Cnvrg
         commit = result["result"]["commit"]
         if !link
           if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-            log_end(0)
 
-            say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+            log_message("Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::BLUE)
             exit(1)
           end
 
-          say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
+          log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
         end
         result = result["result"]["tree"]
         # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
@@ -1672,8 +1712,7 @@ module Cnvrg
         # end
         check = Helpers.checkmark()
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_end(0)
-          say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
+          log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
           return true
         end
         update_count = 0
@@ -1682,77 +1721,133 @@ module Cnvrg
         successful_deletions = []
         if options["verbose"]
           if update_total == 1
-            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
+            log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
           else
-            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
+            log_message("Updating #{update_total} files", Thor::Shell::Color::BLUE)
           end
         else
-          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
+          log_message("Syncing files", Thor::Shell::Color::BLUE, ((options["sync"] or sync)) ? false : true)
 
         end
-
         # Start commit
-
         commit_sha1 = @files.start_commit(new_branch)["result"]["commit_sha1"]
 
         # upload / update
         begin
-          (result["added"] + result["updated_on_local"]).each do |f|
-            absolute_path = "#{@project.local_path}/#{f}"
-            relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-            if File.directory?(absolute_path)
-              resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-              if resDir
-                update_count += 1
-                successful_updates<< relative_path
-              end
-            else
-              res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-              if res
-                update_count += 1
-                successful_updates<< relative_path
-              else
-                say "Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED
 
-                @files.rollback_commit(commit_sha1)
-                log_end(1, "Failed to upload: #{ File.basename(absolute_path) }, rolling back")
-                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-                exit(1)
+          parallel_options = {
+              :progress => {
+                  :title => "Upload Progress",
+                  :progress_mark => '=',
+                  :format => "%b>>%i| %p%% %t",
+                  :starting_at => 0,
+                  :total => update_total,
+                  :autofinish => true
+              },
+              in_processes: ParallelProcesses,
+              in_thread: ParallelThreads,
+              isolation: true
+          }
+          if (result["added"] + result["updated_on_local"]).size > 0
+
+            begin
+              upload_result = Parallel.map((result["added"] + result["updated_on_local"]), parallel_options) do |f|
+                absolute_path = "#{@project.local_path}/#{f}"
+                relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
+                if File.directory?(absolute_path)
+                  resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+                  if resDir
+                    f
+                    # progressbar.increment
+                    update_count += 1
+                    successful_updates<< relative_path
+                  else
+                    log_message("Failed to upload directory: #{ relative_path }", Thor::Shell::Color::RED)
+
+                    raise Parallel::Kill
+                  end
+
+                else
+
+                  res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+                  if res
+                    f
+                    update_count += 1
+
+                    successful_updates<< relative_path
+                  else
+                    log_message("Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED)
+
+                    raise Parallel::Kill
+
+                  end
+                end
               end
+            rescue Interrupt
+              log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
+              @files.rollback_commit(commit_sha1)
+
+              exit(1)
             end
           end
 
+
+          successful_updates = upload_result.to_a
+
           # delete
+
           deleted = update_deleted(result["deleted"])
-          deleted.each do |f|
-            relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-            if relative_path.end_with?("/")
-              if @files.delete_dir(f, relative_path, commit_sha1)
-                # update_count += 1
-                successful_updates<< relative_path
-              end
-            else
-              if @files.delete_file(f, relative_path, commit_sha1)
-                # update_count += 1
-                successful_updates<< relative_path
+          begin
+
+            deleted_result = Parallel.map(deleted, in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
+
+              relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
+              if relative_path.end_with?("/")
+                if @files.delete_dir(f, relative_path, commit_sha1)
+                  f
+                else
+                  log_message("Failed to delete directory: #{ f }", Thor::Shell::Color::RED)
+
+                end
+              else
+                if @files.delete_file(f, relative_path, commit_sha1)
+                  f
+                else
+                  log_message("Failed to delete file: #{ f }", Thor::Shell::Color::RED)
+
+                end
               end
             end
+          rescue Interrupt
+            log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
+            @files.rollback_commit(commit_sha1)
+
+            exit(1)
+
           end
 
+
+          successful_deletions += successful_deletions.select { |x| not x.nil? }
+
+
+          successful_updates = successful_updates.select { |x| not x.nil? }
+
+          update_count = successful_updates.size
+
         rescue SignalException
-          log_end(-1)
           @files.rollback_commit(commit_sha1)
           say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
           exit(0)
         rescue => e
-          log_end(-1, e.message)
           @files.rollback_commit(commit_sha1)
-          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          log_message("Exception while trying to upload, Rolling back", Thor::Shell::Color::RED)
+          log_error(e)
           exit(0)
         end
         if !result["deleted"].nil? and !result["deleted"].empty?
           update_count += result["deleted"].size
         end
+
         if update_count == update_total
           res = @files.end_commit(commit_sha1)
           if (Cnvrg::CLI.is_response_success(res, false))
@@ -1762,9 +1857,9 @@ module Cnvrg
 
               @project.update_idx_with_commit!(commit_sha1)
             rescue => e
-              log_end(-1, e.message)
               @files.rollback_commit(commit_sha1)
-              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
+              log_message("Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED)
+              log_error(e)
               exit(1)
 
             end
@@ -1774,49 +1869,47 @@ module Cnvrg
             end
 
             if options["verbose"]
-              say "#{check} Done", Thor::Shell::Color::BLUE
+              log_message("#{check} Done", Thor::Shell::Color::BLUE)
               if successful_updates.size >0
-                say "Updated:", Thor::Shell::Color::GREEN
+                log_message("Updated:", Thor::Shell::Color::GREEN)
                 suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say suc.join("\n"), Thor::Shell::Color::GREEN
+                log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
               end
               if successful_deletions.size >0
-                say "Deleted:", Thor::Shell::Color::GREEN
+                log_message("Deleted:", Thor::Shell::Color::GREEN)
                 del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say del.join("\n"), Thor::Shell::Color::GREEN
+                log_message(del.join("\n"), Thor::Shell::Color::GREEN)
               end
-              say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+              log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
             else
               if (options["sync"] or sync) and direct
-                say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
+                log_message("#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN)
 
               else
-                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+                log_message("#{check} Changes were updated successfully", Thor::Shell::Color::GREEN)
 
               end
 
             end
 
-            log_end(0)
           else
             @files.rollback_commit(commit_sha1)
-            log_end(1, "error. Rolling Back all changes")
-            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
+            log_message("Error: couldn't commit changes,  Rolling Back all changes.", Thor::Shell::Color::RED)
           end
         else
-          log_end(1, "error. Rolling Back all changes")
-          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
+          log_message("Error: uploaded only: #{update_count} / #{update_total}, \n Rolling back", Thor::Shell::Color::RED)
 
           @files.rollback_commit(commit_sha1)
         end
       rescue => e
-        log_end(-1,e.message)
 
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+        log_error(e)
         @files.rollback_commit(commit_sha1)
+        puts e.message
+
         exit(1)
       rescue SignalException
-        log_end(-1)
 
         say "\nAborting", Thor::Shell::Color::BLUE
         say "\nRolling back all changes", Thor::Shell::Color::BLUE
@@ -1852,7 +1945,7 @@ module Cnvrg
           end
         end
         if !@project.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+          log_message("Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW)
         end
         new_branch = options["new_branch"] || false
 
@@ -1862,8 +1955,7 @@ module Cnvrg
 
         commit = res["commit"]
         if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
-          log_end(0)
+          log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
           return true
         end
         # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
@@ -1879,106 +1971,149 @@ module Cnvrg
         #   exit(1)
         #   end
         update_count = 0
-        update_total = result["updated_on_server"].size + result["conflicts"].size
+        update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
 
 
         successful_changes = []
         if update_total ==1
-          say "Downloading #{update_total} file", Thor::Shell::Color::BLUE unless options["sync"]
+          log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !options["sync"])
         elsif update_total == 0
-          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
-          log_end(0)
+          log_message("Project is up to date", Thor::Shell::Color::GREEN, !options["sync"])
           return true
         elsif options["verbose"]
-          say "Downloading #{update_total} files", Thor::Shell::Color::BLUE
+          log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
         else
-          say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
-
+          log_message("Syncing files", Thor::Shell::Color::BLUE, !options["sync"])
 
         end
+        parallel_options = {
+            :progress => {
+                :title => "Download Progress",
+                :progress_mark => '=',
+                :format => "%b>>%i| %p%% %t",
+                :starting_at => 0,
+                :total => result["updated_on_server"].size,
+                :autofinish => true
+            },
+            in_processes: ParallelProcesses,
+            in_thread: ParallelThreads
+        }
+        if !result["conflicts"].empty?
+          begin
 
-        result["conflicts"].each do |f|
-          relative_path = f.gsub(/^#{@project.local_path}/, "")
-          if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
-            successful_changes << relative_path
-          end
 
-        end
-        result["updated_on_server"].each do |f|
-          relative_path = f.gsub(/^#{@project.local_path}/, "")
-          if f.end_with? "/"
-            # dir
-            if @files.download_dir(f, relative_path, project_home)
-              successful_changes << relative_path
+            conflicts_result = Parallel.map(result["conflicts"], in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
 
+              relative_path = f.gsub(/^#{@project.local_path}/, "")
+              if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
+                f
+              else
+                log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
+                raise Parallel::Kill
+
+              end
             end
-          else
-            # blob
-            if @files.download_file_s3(f, relative_path, project_home)
-              successful_changes << relative_path
+          rescue Interrupt
+
+            log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+            @files.revoke_download(result["conflicts"], [])
+            exit(1)
+          end
+        end
+
+
+        successful_changes += conflicts_result.to_a
+        if !result["updated_on_server"].empty?
+          begin
+            updated_on_server_result = Parallel.map(result["updated_on_server"], parallel_options) do |f|
+
+              relative_path = f.gsub(/^#{@project.local_path}/, "")
+              if f.end_with? "/"
+                # dir
+                if @files.download_dir(f, relative_path, project_home)
+                  f
+                else
+                  log_message("Couldn't dcreate directory: #{f}", Thor::Shell::Color::RED)
+                  raise Parallel::Kill
+
+
+                end
+
+              else
+                # blob
+                if @files.download_file_s3(f, relative_path, project_home)
+                  f
+                else
+
+
+                  log_message("Couldn't download: #{f}", Thor::Shell::Color::RED)
+                  raise Parallel::Kill
+
+
+                end
+              end
+
+
             end
+            successful_changes += updated_on_server_result.to_a
+          rescue Interrupt
+            log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+            @files.revoke_download(result["conflicts"], result["updated_on_server"])
+            exit(1)
+
           end
+        end
+
+        deleted = result["deleted"].to_a
+        delete_res = @files.delete_commit_files_local(deleted)
+        if !delete_res
+          log_message("Couldn't delete #{deleted.join(" ")}", Thor::Shell::Color::RED)
+          log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
+
+          @files.revoke_download(result["conflicts"], result["updated_on_server"])
+          exit(1)
 
         end
+        successful_changes += deleted
+
+
+        successful_changes = successful_changes.select { |x| not x.nil? }
+
         if update_total == successful_changes.size
           # update idx with latest commit
           @project.update_idx_with_commit!(commit)
           check = Helpers.checkmark()
 
           if options["verbose"]
-            say "#{check} Done, Downloaded:", Thor::Shell::Color::GREEN
-            say successful_changes.join("\n"), Thor::Shell::Color::GREEN
-            say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
+            log_message("#{check} Done, Downloaded:", Thor::Shell::Color::GREEN)
+            log_message(successful_changes.join("\n"), Thor::Shell::Color::GREEN)
+            log_message("Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN)
           else
-            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
+            log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
           end
 
 
-          log_end(0)
         end
       rescue => e
-        log_end(-1)
 
-        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
+        log_error(e)
         if successful_changes.nil?
           exit(1)
         end
-        successful_changes.each do |f|
-
-          abs_path = "#{@project.local_path}/#{f}"
-          filename = File.basename abs_path
-          say "revoking #{filename}"
-          if result["conflicts"].include? f
-            @files.revoke_download_file(abs_path, f, filename, true)
-          elsif result["updated_on_server"].include? f
-            if File.directory? abs_path
-              @files.revoke_download_dir(abs_path, f, project_home)
-            else
-              @files.revoke_download_file(project_home, abs_path, filename)
-            end
-          end
+        begin
+          @files.revoke_download(result["conflicts"], result["updated_on_server"])
         end
         exit(1)
       rescue SignalException
-        log_end(-1)
         say "\nAborting", Thor::Shell::Color::BLUE
         if successful_changes.nil?
           exit(1)
         end
-        successful_changes.each do |f|
-
-          abs_path = "#{@project.local_path}/#{f}"
-          filename = File.basename abs_path
-          say "revoking #{filename}"
-          if result["conflicts"].include? f
-            @files.revoke_download_file(abs_path, f, filename, true)
-          elsif result["updated_on_server"].include? f
-            if File.directory? abs_path
-              @files.revoke_download_dir(abs_path, f, project_home)
-            else
-              @files.revoke_download_file(project_home, abs_path, filename)
-            end
-          end
+        begin
+          @files.revoke_download(result["conflicts"], result["updated_on_server"])
         end
         exit(1)
       end
@@ -2014,30 +2149,36 @@ module Cnvrg
           File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
           current_tree = Dir.entries(".").reject { |file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" }
 
-          response["result"]["tree"].each do |f|
+          jump_result = Parallel.map(response["result"]["tree"], in_processes: ParallelProcesses, in_thread: ParallelThreads) do |f|
+
             relative_path = f[0].gsub(/^#{@project.local_path}/, "")
             if f[0].end_with? "/"
               # dir
               if @files.download_dir(f[0], relative_path, project_home)
                 current_tree.delete(relative_path[0, relative_path.size-1])
-                successful_changes << relative_path
+                f[0]
               end
             else
               # blob
               if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=commit_sha1)
                 current_tree.delete(relative_path)
-
-                successful_changes << relative_path
+                f[0]
               end
             end
+
           end
+          successful_changes = jump_result.select { |x| not x.nil? }
         end
+
+
         FileUtils.rm_rf(current_tree)
-        say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
-        log_end(0)
+        log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
       rescue SignalException
-        log_end(-1)
-        exi(1)
+
+        exit(1)
       end
     end
 
@@ -2083,13 +2224,14 @@ module Cnvrg
           end
 
 
-          say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("Done. Jumped to #{commit_sha1} completed successfully", Thor::Shell::Color::GREEN)
         end
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
       rescue SignalException
-        log_end(-1)
-        exi(1)
+        exit(1)
       end
     end
 
@@ -2100,11 +2242,9 @@ module Cnvrg
 
     def sync(direct=true)
       verify_logged_in(true) if direct
-      if options["verbose"]
-        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
-      else
-        say 'Syncing project', Thor::Shell::Color::BLUE
-      end
+      log_start(__method__, args, options)
+      log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
+      log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
 
 
       invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
@@ -2206,6 +2346,7 @@ module Cnvrg
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
     method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+
     def exec(*cmd)
 
       log = []
@@ -2256,7 +2397,7 @@ module Cnvrg
           if image_proj and image_proj.is_docker
             container= image_proj.get_container
             if !container
-              say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
+              log_message("Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED)
               exit(1)
             end
 
@@ -2288,8 +2429,8 @@ module Cnvrg
         begin
           machine_activity = @exp.get_machine_activity(working_dir)
           @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
-          say "Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
-          say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
+          log_message("Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
+          log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -2303,7 +2444,7 @@ module Cnvrg
                     cur_time = Time.now
                     real_time= Time.now-real
 
-                    cur_log = { time: cur_time,
+                    cur_log = {time: cur_time,
                                message: line,
                                type: "stdout",
                                real: real_time
@@ -2311,7 +2452,7 @@ module Cnvrg
                     }
 
 
-                      if print_log
+                    if print_log
                       puts cur_log
                     end
                     log << cur_log
@@ -2323,7 +2464,7 @@ module Cnvrg
                             stats = usage_metrics_in_docker(docker_id)
                             cpu = stats[0]
                             memory = stats[1]
-                            if  is_on_gpu
+                            if is_on_gpu
                               gpu_stats = gpu_util
                               gpu_utilization = gpu_stats[0]
                               gpu_memory_util = gpu_stats[1]
@@ -2334,9 +2475,9 @@ module Cnvrg
                           end
                           log.each do |l|
                             if remote and is_on_gpu
-                              l.merge!(cpu:cpu,memory:memory, gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
+                              l.merge!(cpu: cpu, memory: memory, gpu_util: gpu_utilization, gpu_memory_util: gpu_memory_util)
                             else
-                              l.merge!(cpu:cpu,memory:memory)
+                              l.merge!(cpu: cpu, memory: memory)
                             end
                           end
 
@@ -2351,8 +2492,10 @@ module Cnvrg
                         end
 
                       end
-                    rescue
-                      say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
+                    rescue => e
+
+                      log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
+                      log_error(e)
                     end
 
                   end
@@ -2367,22 +2510,23 @@ module Cnvrg
                   end
 
                 rescue Errno::EIO => e
-                  # break
-                rescue Errno::ENOENT
-                  log_end(1, "command #{cmd} isn't valid")
+                  log_error(e)
+
+                    # break
+                rescue Errno::ENOENT => e
 
                   exp_success = false
 
-                  say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+                  log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
+                  log_error(e)
                 rescue PTY::ChildExited
-                  log_end(1, "proccess exited")
                   exp_success = false
-                  say "The process exited!", Thor::Shell::Color::RED
+                  log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
-                  log_end(-1, e.message)
                   res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
 
-                  say "Error occurred,aborting", Thor::Shell::Color::RED
+                  log_message("Error occurred,aborting", Thor::Shell::Color::RED)
+                  log_error(e)
                   exit(0)
                 end
                 ::Process.wait pid
@@ -2396,21 +2540,19 @@ module Cnvrg
                   if !Cnvrg::Helpers.internet_connection?
                     wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
                     if wait_offline
-                      say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+                      log_message("Waiting until your'e online..", Thor::Shell::Color::BLUE)
                       while !Cnvrg::Helpers.internet_connection?
                       end
                       exit_status = 0
                     else
-                      say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
-                      log_end(1, "experiment has failed,computer is offline")
+                      log_message("Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED)
                       exit(0)
                     end
                   else
 
                     end_commit = @project.last_local_commit
                     res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-                    say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
-                    log_end(1, "experiment has failed")
+                    log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
                     exit(0)
                   end
 
@@ -2427,15 +2569,15 @@ module Cnvrg
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                 check = Helpers.checkmark()
-                say "#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN
-                log_end(0)
+                log_message("#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN)
               end
             rescue => e
-              log_end(-1, e.message)
               if container
                 container.stop()
               end
-              say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
+              log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
+              log_error(e)
+
               exit(1)
             end
 
@@ -2445,7 +2587,6 @@ module Cnvrg
         end
       rescue SignalException
         exit_status = -1
-        log_end(-1)
         end_commit = @project.last_local_commit
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
@@ -2488,7 +2629,7 @@ module Cnvrg
         start_commit = @project.last_local_commit
         cmd = cmd.join("\s")
 
-        say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
+        log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
 
         @exp = Experiment.new(@project.owner, @project.slug)
         machine_activity = @exp.get_machine_activity(project_home)
@@ -2540,7 +2681,7 @@ module Cnvrg
 
                       end
                     rescue
-                      say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
+                      log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
                     end
 
                   end
@@ -2557,18 +2698,16 @@ module Cnvrg
                 rescue Errno::EIO => e
                   break
                 rescue Errno::ENOENT
-                  log_end(1, "command #{cmd} isn't valid")
 
                   exp_success = false
 
-                  say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+                  log_message("command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED)
                 rescue PTY::ChildExited
-                  log_end(1, "proccess exited")
                   exp_success = false
-                  say "The process exited!", Thor::Shell::Color::RED
+                  log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
-                  log_end(-1, e.message)
-                  say "Error occurred, aborting", Thor::Shell::Color::RED
+                  log_message("Error occurred, aborting", Thor::Shell::Color::RED)
+                  log_error(e)
                   exit(0)
                 end
                 ::Process.wait pid
@@ -2582,13 +2721,12 @@ module Cnvrg
                   if !Cnvrg::Helpers.internet_connection?
                     wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
                     if wait_offline
-                      say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+                      log_message("Waiting until your'e online..", Thor::Shell::Color::BLUE)
                       while !Cnvrg::Helpers.internet_connection?
                       end
                       exit_status = 0
                     else
-                      say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
-                      log_end(1, "experiment has failed,computer is offline")
+                      log_message("Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED)
                       exit(0)
                     end
                   else
@@ -2596,18 +2734,17 @@ module Cnvrg
                     end_commit = @project.last_local_commit
                     res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                     @image.update_image_activity(@project.last_local_commit, @exp.slug)
-                    say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
-                    log_end(1, "experiment has failed")
+                    log_message("Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED)
                     exit(0)
                   end
 
                 end
                 if sync_after
-                  say "Syncing project after running", Thor::Shell::Color::BLUE
+                  log_message("Syncing project after running", Thor::Shell::Color::BLUE)
                   # Sync after run
                   download()
                   upload()
-                  say "Done Syncing", Thor::Shell::Color::BLUE
+                  log_message("Done Syncing", Thor::Shell::Color::BLUE)
                 end
                 end_commit = @project.last_local_commit
 
@@ -2615,12 +2752,11 @@ module Cnvrg
                 @image.update_image_activity(@project.last_local_commit, @exp.slug)
 
                 check = Helpers.checkmark()
-                say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
-                log_end(0)
+                log_message("#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
               end
             rescue => e
-              log_end(-1, e.message)
-              say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
+              log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
+              log_error(e)
               exit(1)
             end
 
@@ -2628,9 +2764,12 @@ module Cnvrg
           end
 
         end
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
+
       rescue SignalException
         exit_status = -1
-        log_end(-1)
         end_commit = @project.last_local_commit
         if !@exp.nil?
 
@@ -2684,7 +2823,7 @@ module Cnvrg
           elsif time[2].downcase().start_with? "days"
             new = local + (time[1].to_i * 3600 * 24)
           else
-            say "Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED
+            log_message("Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED)
             exit(1)
           end
           new_time = new.to_s
@@ -2738,13 +2877,13 @@ module Cnvrg
           image_slug = image.image_slug
         end
 
-        invoke :sync, [false], []  if sync_before
+        invoke :sync, [false], [] if sync_before
 
 
         if command.include? "'"
           oc = command.to_enum(:scan, /'/).map { Regexp.last_match }
           pairs = oc.enum_for(:each_slice, 2).to_a
-          pairs.each_with_index do |p,i|
+          pairs.each_with_index do |p, i|
             add=0
             if i!=0
               add=2*i
@@ -2755,7 +2894,7 @@ module Cnvrg
 
 
         end
-        say "Running remote experiment", Thor::Shell::Color::BLUE
+        log_message("Running remote experiment", Thor::Shell::Color::BLUE)
         exp = Experiment.new(project.owner, project.slug)
         res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
@@ -2796,16 +2935,18 @@ module Cnvrg
             str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
           end
 
-          say str, Thor::Shell::Color::GREEN
+          log_message(str, Thor::Shell::Color::GREEN)
 
           exit(0)
           # end
         end
 
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
       rescue SignalException
         exit_status = -1
-        log_end(-1)
         end_commit = project.last_local_commit
 
         res = @exp.end(log, exit_status, end_commit, "", "")
@@ -2852,7 +2993,7 @@ module Cnvrg
         begin
           num_workers = workers.to_i
         rescue
-          say "Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED
+          log_message("Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED)
           exit(1)
         end
         file_as_input = options["file_as_input"] || false
@@ -2910,16 +3051,18 @@ module Cnvrg
           #   end
           # else
           check = Helpers.checkmark()
-          say "#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/deploys/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN
+          log_message("#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/deploys/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN)
 
           exit(0)
           # end
         end
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
 
       rescue SignalException
         exit_status = -1
-        log_end(-1)
         end_commit = project.last_local_commit
 
         res = @exp.end(log, exit_status, end_commit, "", "")
@@ -3017,7 +3160,7 @@ module Cnvrg
         res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
           if res["result"]["machine"] == -1
-            say "There are no available machines", Thor::Shell::Color::BLUE
+            log_message("There are no available machines", Thor::Shell::Color::BLUE)
             create = yes? "create new machine?", Thor::Shell::Color::YELLOW
             if create
               res = Cnvrg::API.request("users/#{@image.owner}/machines/list", 'GET')
@@ -3030,22 +3173,22 @@ module Cnvrg
                     if !url.nil? and !url.empty?
                       check = Helpers.checkmark()
 
-                      say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
+                      log_message("#{check} Notebook server started successfully", Thor::Shell::Color::GREEN)
                       Launchy.open(url)
                     else
-                      say "Couldn't run notebook server", Thor::Shell::Color::RED
+                      log_message("Couldn't run notebook server", Thor::Shell::Color::RED)
                     end
                     exit(0)
                   end
                 end
               else
-                say "No machines are avilable", Thor::Shell::Color::RED
+                log_message("No machines are avilable", Thor::Shell::Color::RED)
                 exit(0)
               end
 
 
             else
-              say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
+              log_message("Can't execute command on remote machine with local image", Thor::Shell::Color::RED)
               exit(1)
 
             end
@@ -3053,17 +3196,18 @@ module Cnvrg
             note_url = res["result"]["notebook_url"]
             @image.set_note_url(note_url)
             check = Helpers.checkmark()
-            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
+            log_message("#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN)
             # Launchy.open(url)
 
             exit(0)
           end
         end
-
+      rescue => e
+        log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
       rescue SignalException
         exit_status = -1
-        log_end(-1)
         end_commit = @project.last_local_commit
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
@@ -3073,7 +3217,7 @@ module Cnvrg
       end
     end
 
-    desc 'search_libraries', 'search if  libraries installed'
+    desc 'search_libraries', 'search if  libraries installed', :hide => true
 
     def search_libraries(library)
       begin
@@ -3135,7 +3279,7 @@ module Cnvrg
       end
     end
 
-    desc 'show_libraries', 'show system libraries installed'
+    desc 'show_libraries', 'show system libraries installed', :hide => true
     method_option :system, :type => :boolean, :aliases => ["-s", "--s"], :default => false, :desc => "show also system libraries installed"
 
     def show_libraries
@@ -3194,7 +3338,7 @@ module Cnvrg
     end
 
 
-    desc 'run_notebook', 'Starts a new notebook environment'
+    desc 'run_notebook', 'Starts a new notebook environment', :hide => true
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
@@ -3343,7 +3487,7 @@ module Cnvrg
 
       rescue => e
         log_end(-1, e.message)
-        say "Error occurd, aborting", Thor::Shell::Color::RED
+        say "Error occurred, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
@@ -3380,10 +3524,10 @@ module Cnvrg
         project_dir = is_cnvrg_dir()
 
 
-        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
+        log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
         invoke :sync, [false], :verbose => options["verbose"]
 
-        say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
+        log_message("Done Syncing", Thor::Shell::Color::BLUE, options["verbose"])
 
 
         @project = Project.new(project_dir)
@@ -3402,37 +3546,34 @@ module Cnvrg
           if !container
             check = Helpers.checkmark()
 
-            say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
+            log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
             exit(0)
           end
 
 
-          say "Stopping notebook server...", Thor::Shell::Color::BLUE
+          log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
 
           check = Helpers.checkmark()
           image.remove_note_slug()
           container.stop()
 
-          say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
 
           exit(0)
         elsif remote
-          say "Stopping notebook server...", Thor::Shell::Color::BLUE
+          log_message("Stopping notebook server...", Thor::Shell::Color::BLUE)
           check = Helpers.checkmark()
 
-          say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
-          log_end(0)
+          log_message("#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN)
           exit(0)
         end
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurd, aborting"
+        log_message("Error occurd, aborting", Thor::Shell::Color::RED)
+        log_error(e)
         if container
           container.stop()
         end
       rescue SignalException
-        log_end(-1)
         say "Aborting"
         exit(1)
       end
@@ -3441,7 +3582,7 @@ module Cnvrg
     end
 
 
-    desc 'install_system_libraries', 'Install libraries'
+    desc 'install_system_libraries', 'Install libraries', :hide => true
 
     def install_system_libraries(*command_to_run)
       begin
@@ -3476,7 +3617,7 @@ module Cnvrg
         log_end(0)
       rescue => e
         log_end(-1, e.message)
-        say "Error occurd, aborting"
+        say "Error occurred, aborting"
         if container
           container.stop()
         end
@@ -3491,7 +3632,7 @@ module Cnvrg
 
     end
 
-    desc 'install_libraries', 'Install libraries'
+    desc 'install_libraries', 'Install libraries', :hide => true
     method_option :requirement, :type => :string, :aliases => ["-r", "--r"], :default => "", :desc => "Install from the given requirements file"
 
     def install_python_libraries(*lib)
@@ -3538,7 +3679,7 @@ module Cnvrg
         log_end(0)
       rescue => e
         log_end(-1, e.message)
-        say "Error occurd, aborting"
+        say "Error occurred, aborting"
         if container
           container.stop()
         end
@@ -3558,121 +3699,117 @@ module Cnvrg
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
-    method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
-    method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+    method_option :base, :type => :boolean, :aliases => ["-b", "--base"], :default => false
     method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
     method_option :docker_path, :type => :string, :aliases => ["--docker_path"], :default => ""
 
 
-    desc 'create_custom_image', 'run commands inside containers', :hide=>true
+    desc 'create_custom_image', 'run commands inside containers', :hide => true
 
     def build_image(image_name)
       begin
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                   "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
-      instance_type = get_instance_type(instances)
-      image_extend = options["image"]
-      public = options["public"]
-      base = options["base"]
-      python3 = options["python3"]
-      docker_path = options["docker_path"]
-      owner = CLI.get_owner
-      checks = Helpers.checkmark()
-      tar_path = nil
-      if !docker_path.nil? and !docker_path.empty?
-        docker_path  = File.absolute_path(docker_path)
-        #create tar of the docker path: it could be a docker file, and it could be a docker folder
-        tar_path = File.expand_path('~')+"/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
-        resp = create_docker_tar(docker_path,tar_path)
-        if !resp
-          say "Couldn't create tar from docker path",  Thor::Shell::Color::RED
-          FileUtils.rm_rf tar_path
-          exit(1)
-        end
-        files = Cnvrg::Files.new(owner, "")
-        resp = Images.create_new_custom_image_with_docker(instance_type,owner,image_name,public,base,image_extend,python3,tar_path,files)
-        if resp
-        end
-      else
-        say "Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE
-        resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3,nil)
-
-      end
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+        instance_type = get_instance_type(instances)
+        image_extend = options["image"]
+        public = options["public"]
+        base = options["base"]
+        python3 = options["python3"]
+        docker_path = options["docker_path"]
+        owner = CLI.get_owner
+        checks = Helpers.checkmark()
+        tar_path = nil
+        if !docker_path.nil? and !docker_path.empty?
+          docker_path = File.absolute_path(docker_path)
+          #create tar of the docker path: it could be a docker file, and it could be a docker folder
+          tar_path = File.expand_path('~')+"/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
+          resp = create_docker_tar(docker_path, tar_path)
+          if !resp
+            log_message("Couldn't create tar from docker path", Thor::Shell::Color::RED)
+            FileUtils.rm_rf tar_path
+            exit(1)
+          end
+          files = Cnvrg::Files.new(owner, "")
+          resp = Images.create_new_custom_image_with_docker(instance_type, owner, image_name, public, base, image_extend, python3, tar_path, files)
+          if resp
+          end
+        else
+          log_message("Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE)
+          resp = Images.create_new_custom_image(instance_type, owner, image_name, public, base, image_extend, python3, nil)
 
-      if Cnvrg::CLI.is_response_success(resp,false)
-        image_slug = resp["result"]["slug"]
-        container = resp["result"]["machine_c"]
-        say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
-        say "Connecting to machine", Thor::Shell::Color::BLUE
-        ssh = Ssh.new(resp)
-        if !ssh.is_ssh
-          say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
-          Images.revoke_custom_new_image(owner, image_slug)
-          log_end(-1,"Couldn't connect to machine,aborting")
         end
-        say "run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE
-        begin
-          logs = []
 
-          while true
-            command = ask("$>")
-            logs <<  { time: Time.now,
-                                 message: command,
-                                 type: "stdout"
-            }
-            if command.eql? "quit"
-              say "Commiting Image..", Thor::Shell::Color::BLUE
-              break
-            end
-            res = ssh.exec_command(command)
-            begin
-              res_parsed = JSON.parse(res)
-              res = res_parsed.join(",")
-            end
+        if Cnvrg::CLI.is_response_success(resp, false)
+          image_slug = resp["result"]["slug"]
+          container = resp["result"]["machine_c"]
+          log_message("#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN)
+          log_message("Connecting to machine", Thor::Shell::Color::BLUE)
+          ssh = Ssh.new(resp)
+          if !ssh.is_ssh
+            log_message("Couldn't connect to machine,aborting", Thor::Shell::Color::RED)
+            Images.revoke_custom_new_image(owner, image_slug)
+          end
+          log_message("run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE)
+          begin
+            logs = []
 
-            puts res
-            logs <<  { time: Time.now,
+            while true
+              command = ask("$>")
+              logs << {time: Time.now,
+                       message: command,
+                       type: "stdout"
+              }
+              if command.eql? "quit"
+                log_message("Commiting Image..", Thor::Shell::Color::BLUE)
+                break
+              end
+              res = ssh.exec_command(command)
+              begin
+                res_parsed = JSON.parse(res)
+                res = res_parsed.join(",")
+              end
+
+              puts res
+              logs << {time: Time.now,
                        message: res,
                        type: "stdout"
-            }
-            logs.flatten!
+              }
+              logs.flatten!
 
-          end
+            end
 
-        rescue SignalException
-          say "Commiting Image..", Thor::Shell::Color::BLUE
+          rescue SignalException
+            log_message("Commiting Image..", Thor::Shell::Color::BLUE)
 
-        end
-        resp = Images.commit_custom_image(owner,image_slug,logs)
-        if Cnvrg::CLI.is_response_success(resp,false)
-          say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
-        else
-          if image_slug
-            Images.revoke_custom_new_image(owner, image_slug)
+          end
+          resp = Images.commit_custom_image(owner, image_slug, logs)
+          if Cnvrg::CLI.is_response_success(resp, false)
+            log_message("#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN)
+          else
+            if image_slug
+              Images.revoke_custom_new_image(owner, image_slug)
+            end
+            if ssh
+              ssh.close_ssh()
+            end
+            log_message("Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED)
+
+            exit(1)
           end
           if ssh
             ssh.close_ssh()
           end
-          say "Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED
-
-          exit(1)
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-
-
 
 
+        end
+      rescue => e
+        log_message("Error occurd, aborting", Thor::Shell::Color::RED)
 
-      end
-      rescue =>e
-        puts e
+        log_error(e)
         if image_slug
           Images.revoke_custom_new_image(owner, image_slug)
         end
@@ -3695,7 +3832,7 @@ module Cnvrg
     end
 
 
-    desc 'build', 'run commands inside containers'
+    desc 'build', 'run commands inside containers', :hide => true
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
     def build(*cmd)
@@ -3756,7 +3893,7 @@ module Cnvrg
         log_end(0)
       rescue => e
         log_end(-1, e.message)
-        say "Error occurd, aborting", Thor::Shell::Color::RED
+        say "Error occurred, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
@@ -3771,7 +3908,7 @@ module Cnvrg
 
     end
 
-    desc 'commit_notebook', 'commit notebook changes to create a new notebook image'
+    desc 'commit_notebook', 'commit notebook changes to create a new notebook image', :hide => true
 
     def commit_image
       verify_logged_in(true)
@@ -3804,7 +3941,7 @@ module Cnvrg
         return new_image.id
       rescue => e
         log_end(-1, e.message)
-        say "\nError occurred,aborting"
+        say "\nError occurred, aborting"
         exit(1)
       rescue SignalException
         log_end(-1)
@@ -3813,7 +3950,7 @@ module Cnvrg
       end
     end
 
-    desc 'sync_image', 'sync current container into image, and push it to cnvrg repository'
+    desc 'sync_image', 'sync current container into image, and push it to cnvrg repository', :hide => true
     method_option :is_public, :type => :boolean, :aliases => ["-p", "--p"], :default => false, :desc => "is public"
     method_option :is_base, :type => :boolean, :aliases => ["-b", "--b"], :default => false, :desc => "is base for others images"
     method_option :message, :type => :string, :aliases => ["-m", "--m"], :default => "", :desc => "commit message for this image"
@@ -3854,13 +3991,13 @@ module Cnvrg
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir()
       if !name.empty? and name == "cnvrg"
-        say "can't create image with the name cnvrg", Thor::Shell::Color::RED
+        log_message("can't create image with the name cnvrg", Thor::Shell::Color::RED)
         exit(1)
       end
       begin
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
-          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          log_message("Couldn't find image related to project", Thor::Shell::Color::RED)
           exit(0)
         end
         if !name.nil? and !name.empty?
@@ -3870,18 +4007,18 @@ module Cnvrg
         end
         stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
         if stored_commands.nil? or stored_commands.empty?
-          say "Nothing to push", Thor::Shell::Color::BLUE
+          log_message("Nothing to push", Thor::Shell::Color::BLUE)
           exit(0)
         end
 
-        say "Pushing new image", Thor::Shell::Color::BLUE
+        log_message("Pushing new image", Thor::Shell::Color::BLUE)
         if image.create_custom_image(name, working_dir, stored_commands)
 
-          say "Image was updated successfully", Thor::Shell::Color::GREEN
+          log_message("Image was updated successfully", Thor::Shell::Color::GREEN)
         end
       rescue => e
-        log_end(-1, e.message)
-        say "error occurred, aborting"
+        log_message("error occurred, aborting", Thor::Shell::Color::RED)
+        log_error(e)
 
       end
     end
@@ -3903,16 +4040,15 @@ module Cnvrg
         message = message.join(" ")
       end
 
-      say "Saving image's current state", Thor::Shell::Color::BLUE
+      log_message("Saving image's current state", Thor::Shell::Color::BLUE)
       image.save(path)
 
       begin
-        say "Compressing image file to upload", Thor::Shell::Color::BLUE
+        log_message("Compressing image file to upload", Thor::Shell::Color::BLUE)
         gzipRes = system("gzip -f #{path}")
         if !gzipRes
-          log_End(1, "can't create gz file from image")
 
-          say "Couldn't create tar file from image", Thor::Shell::Color::RED
+          log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
           exit(1)
         end
         path = path+".gz"
@@ -3920,7 +4056,7 @@ module Cnvrg
 
         exit_status = $?.exitstatus
         if exit_status == 0
-          say "Uploading image file", Thor::Shell::Color::BLUE
+          log_message("Uploading image file", Thor::Shell::Color::BLUE)
 
           diff =container_changes(Dir.pwd)
           res = @files.upload_image(path, image_name, owner, is_public, is_base, diff[1], diff[0], diff[2], message, image.commit_id)
@@ -3930,23 +4066,19 @@ module Cnvrg
             image_loc.update_slug(res["result"]["id"])
 
             checks = Helpers.checkmark()
-            say "#{checks} Done", Thor::Shell::Color::GREEN
-            log_end(0)
+            log_message("#{checks} Done", Thor::Shell::Color::GREEN)
           else
-            say "Couldn't upload image", Thor::Shell::Color::RED
-            log_end(1, "can't create upload imag")
+            log_message("Couldn't upload image", Thor::Shell::Color::RED)
 
           end
         else
-          say "Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED
-          log_end(1, "can't create upload imag")
+          log_message("Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED)
           exit(1)
         end
       rescue => e
-        log_end(-1, e.message)
-        say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
+        log_message("Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED)
+        log_error(e)
       rescue SignalException
-        log_end(-1)
 
         say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
         exit(1)
@@ -4026,7 +4158,7 @@ module Cnvrg
                 '8888/tcp' => {},
             },
             'HostConfig' => {
-                'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker" ],
+                'Binds' => ["/var/run/docker.sock:/var/run/docker.sock", "/usr/bin/docker:/usr/bin/docker"],
                 'PortBindings' => {
                     '8888/tcp' => [
                         {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
@@ -4264,14 +4396,13 @@ module Cnvrg
         path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.tar.gz"
         @files = Cnvrg::Files.new(owner, "")
 
-        say "Downloading image file", Thor::Shell::Color::BLUE
+        log_message("Downloading image file", Thor::Shell::Color::BLUE)
         begin
           if @files.download_image(path, image_slug, owner)
             gzipRes = system("gunzip -f #{path}")
             if !gzipRes
-              log_End(1, "can't unzip gz file from image")
 
-              say "Couldn't create tar file from image", Thor::Shell::Color::RED
+              log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
               exit(1)
             else
               path = path.gsub(".gz", "")
@@ -4279,17 +4410,14 @@ module Cnvrg
             end
 
           else
-            say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
-            log_end(1, "can't download image")
+            log_message("Couldn't download image #{image_name}", Thor::Shell::Color::RED)
             return false
           end
         rescue Interrupt
-          log_end(-1)
           say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
           exit(1)
         end
       rescue SignalException
-        log_end(-1)
         say "\nAborting"
         exit(1)
       end
@@ -4363,10 +4491,8 @@ module Cnvrg
             printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
           end
         end
-        log_end(0)
         return res["result"]["images"]
       rescue SignalException
-        log_end(-1)
         say "\nAborting"
         exit(1)
       end
@@ -4390,10 +4516,8 @@ module Cnvrg
             printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at
           end
         end
-        log_end(0)
         return res["result"]["images"]
       rescue SignalException
-        log_end(-1)
         say "\nAborting"
         exit(1)
       end
@@ -4401,7 +4525,7 @@ module Cnvrg
 
     end
 
-    desc 'get_machine', 'create new aws machine'
+    desc 'get_machine', 'create new aws machine', :hide => true
 
     def get_machine()
       begin
@@ -4449,12 +4573,12 @@ module Cnvrg
         owner = Cnvrg::CLI.get_owner()
         image = Cnvrg::Images.image_exist(owner, image_name)
         if !image
-          say "Couldn't find image in cnvrg repository", Thor::Shell::Color::RED
+          log_message("Couldn't find image in cnvrg repository", Thor::Shell::Color::RED)
           exit(1)
         end
         path = download_image(image_name, image["slug"])
         if path
-          say "Building image", Thor::Shell::Color::BLUE
+          log_message("Building image", Thor::Shell::Color::BLUE)
           Docker.options[:read_timeout]=216000
           image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
             begin
@@ -4470,20 +4594,16 @@ module Cnvrg
           if not image.nil?
             FileUtils.rm_rf(path)
             checks = Helpers.checkmark()
-            say "#{checks} Image built successfully", Thor::Shell::Color::GREEN
-            log_end(0)
+            log_message("#{checks} Image built successfully", Thor::Shell::Color::GREEN)
             return image
-            log_end(0)
           else
 
-            say "Could not build image", Thor::Shell::Color::RED
-            log_end(1, "Could build  image")
+            log_message("Could not build image", Thor::Shell::Color::RED)
             return false
           end
         else
 
-          say "Could not download image", Thor::Shell::Color::RED
-          log_end(1, "Could build  image")
+          log_message("Could not download image", Thor::Shell::Color::RED)
           return false
 
 
@@ -4511,12 +4631,11 @@ module Cnvrg
           # end
       rescue => e
 
-
-        say "Couldn't build image", Thor::Shell::Color::RED
+        log_message "Error: couldn't build image", Thor::Shell::Color::RED
+        log_error(e)
 
       rescue SignalException
         say "\nAborting"
-        log_end(-1)
         exit(1)
       ensure
         if path
@@ -4543,22 +4662,22 @@ module Cnvrg
         if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
           image = pull(docker_image)
           if image
-            say "downloaded image: #{docker_image}"
+            log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
             @image = Images.new(working_dir, docker_image)
           else
-            say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+            log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
             exit(1)
           end
         else
-          say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+          log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
           exit(1)
 
         end
       elsif docker_image_local.size == 1
-        say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+        log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
         @image = Images.new(working_dir, docker_image_local[0])
       elsif docker_image_local.size >1
-        say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+        log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
         image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
         image_name = image_name.strip
         @image = Images.new(working_dir, image_name)
@@ -4619,11 +4738,11 @@ module Cnvrg
             url = "https://cnvrg.io/api"
           end
           config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-          compression_path =  "#{home_dir}/.cnvrg/tmp"
+          compression_path = "#{home_dir}/.cnvrg/tmp"
           if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
             compression_path = config.to_h[:compression_path]
           end
-          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
+          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url, compression_path: compression_path}
 
           File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
           return true
@@ -4668,13 +4787,48 @@ module Cnvrg
 
       def log_start(command, args="", options={})
         begin
-          $LOG.info "----"
           $LOG.info ruby_version: RUBY_VERSION, os: Cnvrg::Helpers.os(), cli_version: Cnvrg::VERSION
           $LOG.info command: command, args: args, options: options
         rescue
         end
       end
 
+      def log_message(message, type, to_print=true)
+        if to_print
+          say message, type
+        end
+        begin
+          if type == Thor::Shell::Color::BLUE
+            $LOG.info message: message, type: "info"
+          elsif type == Thor::Shell::Color::RED
+            $LOG.error message: message, type: "error"
+          elsif type == Thor::Shell::Color::YELLOW
+            $LOG.warn message: message, type: "warning"
+          elsif type == Thor::Shell::Color::GREEN
+            $LOG.info message: message, type: "success"
+          else
+            $LOG.info message: message, type: "unknown"
+          end
+
+        rescue => e
+          puts e
+        end
+      end
+
+      def log_error(e)
+        begin
+          $LOG.error message: e.message, type: "error"
+          size = e.backtrace.size
+          min = 4
+          if min>= size
+            min = size
+          end
+          $LOG.error message: "Error: #{e.message}, Backtrace: #{e.backtrace[0..min].join(",")}", type: "error"
+        rescue
+        end
+      end
+
+
       def log_end(exit_status=0, error=nil)
         begin
           if exit_status != 0
@@ -4690,6 +4844,8 @@ module Cnvrg
         if response.nil?
           if !Cnvrg::Helpers.internet_connection?
             say("<%= color('Error:You seems to be offline', RED) %>")
+            $LOG.error message: "offline connection", type: "error"
+
           end
           if should_exit
             exit(1)
@@ -4702,6 +4858,7 @@ module Cnvrg
           if response["status"] == 500
             say("<%= color('Server Error', RED) %>")
           else
+            $LOG.error message: error, type: "error"
             say("<%= color('Error:#{error}', RED) %>")
           end
 
@@ -4734,7 +4891,8 @@ module Cnvrg
         api = config.to_h[:api]
         return api.gsub!("/api", "")
       end
-      def  get_compression_path
+
+      def get_compression_path
         home_dir = File.expand_path('~')
 
         config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
@@ -4777,6 +4935,14 @@ module Cnvrg
       def log_handler
         begin
           date = DateTime.now.strftime("%m_%d_%Y")
+          home_dir = File.expand_path('~')
+
+          if !File.directory? home_dir+"/.cnvrg"
+            FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
+          end
+          if !File.exist?(home_dir+"/.cnvrg/config.yml")
+            FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
+          end
           logfile = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
           if !File.exist? logfile
             FileUtils.touch([logfile])
@@ -4798,27 +4964,34 @@ module Cnvrg
             end
 
           end
-          $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
+          config = LogStashLogger.configure do |config|
+            config.customize_event do |event|
+              event.remove('@version')
+              event.remove('severity')
+            end
+          end
+          $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true, config: config)
           remove_old_log_files()
+        rescue
+
         end
       end
 
       def remove_old_log_files()
         begin
-        last_week = (Time.now - (7*24*60*60)).strftime("%Y-%m-%d")
-        home = File.expand_path('~')
-        log_files  = Dir["#{home}/.cnvrg/tmp/*.log"]
-        log_files.each do |l|
-          if File.mtime(l).strftime("%Y-%m-%d") < last_week
-            FileUtils.rm_rf(l)
+          last_week = (Time.now - (7*24*60*60)).strftime("%Y-%m-%d")
+          home = File.expand_path('~')
+          log_files = Dir["#{home}/.cnvrg/tmp/*.log"]
+          log_files.each do |l|
+            if File.mtime(l).strftime("%Y-%m-%d") < last_week
+              FileUtils.rm_rf(l)
+            end
           end
         end
-        end
 
       end
 
 
-
       def verify_logged_in(in_dir=true)
         log_handler()
         auth = Cnvrg::Auth.new
@@ -5109,15 +5282,16 @@ module Cnvrg
           return relative
         end
       end
-      def create_docker_tar(docker_path,tar_path)
+
+      def create_docker_tar(docker_path, tar_path)
         begin
 
-        if File.directory? (docker_path)
-          `cd #{docker_path} && tar -czf #{tar_path} . `
-        else
-          dir_name = File.dirname docker_path
-          `cd #{dir_name} &&  tar -czf #{tar_path} #{File.basename(docker_path)}`
-        end
+          if File.directory? (docker_path)
+            `cd #{docker_path} && tar -czf #{tar_path} . `
+          else
+            dir_name = File.dirname docker_path
+            `cd #{dir_name} &&  tar -czf #{tar_path} #{File.basename(docker_path)}`
+          end
         rescue => e
           puts "Exception while compressing docker path: #{e.message}"
         end
@@ -5125,7 +5299,7 @@ module Cnvrg
         return $?.success?
       end
 
-      def create_tar(path_in, path_out, tar_files,no_compression=false)
+      def create_tar(path_in, path_out, tar_files, no_compression=false)
         #The cd is meant for cases when running cnvrg data uplaod not in the main folder
         begin
           if no_compression
@@ -5144,27 +5318,29 @@ module Cnvrg
         `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
         return $?.success?
       end
+
       def cpu_usage
         cpu_usage = 0.0
         begin
           cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
-        if !cpu.nil?
-          cpu_usage = cpu.to_f
-        end
+          if !cpu.nil?
+            cpu_usage = cpu.to_f
+          end
         rescue
           cpu_usage = 0.0
         end
 
         return cpu_usage
       end
+
       def memory_usage
         prec = 0.0
         begin
-        used = `free -mt |grep Mem: |awk '{print $3}'`
-        total = `free -mt |grep Mem: |awk '{print $2}'`
+          used = `free -mt |grep Mem: |awk '{print $3}'`
+          total = `free -mt |grep Mem: |awk '{print $2}'`
 
-        used_f = used.to_f if !used.nil?
-        total_f = total.to_f if !total.nil?
+          used_f = used.to_f if !used.nil?
+          total_f = total.to_f if !total.nil?
           prec = (used_f / total_f)*100
           prec = prec.round(2)
         rescue
@@ -5174,18 +5350,19 @@ module Cnvrg
 
 
       end
+
       def gpu_util
         gpu = [0.0, 0.0]
         begin
-        gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
+          gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
 
-        if !gpu_stats.nil?
-          gpu = gpu_stats.strip
-          gpu = gpu_stats.gsub!("%","").split(",")
-          gpu[0] = gpu[0].to_f
-          gpu[1] = gpu[1].to_f
-          return gpu
-        end
+          if !gpu_stats.nil?
+            gpu = gpu_stats.strip
+            gpu = gpu_stats.gsub!("%", "").split(",")
+            gpu[0] = gpu[0].to_f
+            gpu[1] = gpu[1].to_f
+            return gpu
+          end
 
         rescue
           return gpu
@@ -5193,18 +5370,19 @@ module Cnvrg
 
 
       end
+
       def usage_metrics_in_docker(docker_id)
-        res = [0.0,0.0]
+        res = [0.0, 0.0]
         begin
           if docker_id.nil?
             docker_id = `cat /etc/hostname`
           end
-            stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
+          stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
           if !stats.nil?
             conv = stats.split(",")
-            cpu = conv[0].gsub!("%","").to_f
-            memory = conv[1].gsub!("%","").to_f
-            res = [cpu,memory]
+            cpu = conv[0].gsub!("%", "").to_f
+            memory = conv[1].gsub!("%", "").to_f
+            res = [cpu, memory]
             return res
           end
         rescue
@@ -5212,7 +5390,6 @@ module Cnvrg
         end
 
 
-
       end
 
       def with_progress_bar