cnvrg 0.0.15 → 0.0.140

data/lib/cnvrg/cli.rb

@@ -7,7 +7,7 @@ require 'uri'
 require 'open-uri'
 require 'json'
 require 'yaml'
-require 'digest' # sha1up
+require 'digest' # sha1
 require "highline/import"
 require 'socket'
 include Open4
@@ -18,10 +18,6 @@ require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
 require 'cnvrg/Images'
-require 'cnvrg/dataset'
-require 'cnvrg/datafiles'
-require 'cnvrg/data'
-require 'cnvrg/ssh'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -33,116 +29,28 @@ require 'fileutils'
 require 'zip'
 require 'active_support/all'
 require 'thor'
-require 'pathname'
-require 'enumerator'
-
-class Thor
-  module Base
-    def initialize(args = [], local_options = {}, config = {})
-      parse_options = Thor.class_options
-
-      # The start method splits inbound arguments at the first argument
-      # that looks like an option (starts with - or --). It then calls
-      # new, passing in the two halves of the arguments Array as the
-      # first two parameters.
-
-      command_options = config.delete(:command_options) # hook for start
-      parse_options = parse_options.merge(command_options) if command_options
-      if local_options.is_a?(Array)
-        array_options = local_options
-        hash_options = {}
-      else
-        # Handle the case where the class was explicitly instantiated
-        # with pre-parsed options.
-        array_options = []
-        hash_options = local_options
-
-      end
-
-      # Let Thor::Options parse the options first, so it can remove
-      # declared options from the array. This will leave us with
-      # a list of arguments that weren't declared.
-      stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
-      opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
-      real_options = []
-
-      real_args = [].replace(array_options)
-      if local_options.is_a? (Array) and !local_options.empty? and args.empty?
-        array_options.each_with_index do |p, i|
-          opt = p
-          if p.include? "="
-            opt = p.split("=")[0]
-          end
-          option = is_option(parse_options.values, opt)
-          if !option
-            break
-          else
-            real_options << p
-            real_args.delete(p)
-            if !p.include? "=" and option.type != :boolean
-              if i+1< array_options.size
-                real_options << array_options[i+1]
-                real_args.delete(array_options[i+1])
-              end
-            end
-
-          end
-
-        end
-
-        args = real_args
-      else
-        if !args.empty? and local_options.is_a? Array and !local_options.empty?
-          args = args + local_options
-        else
-          args = args.flatten()
-        end
-
-      end
-
-
-      self.options = opts.parse(real_options)
-      self.options = config[:class_options].merge(options) if config[:class_options]
-
-      # If unknown options are disallowed, make sure that none of the
-      # remaining arguments looks like an option.
-      opts.check_unknown! if Thor.check_unknown_options?(config)
-
-      # Add the remaining arguments from the options parser to the
-      # arguments passed in to initialize. Then remove any positional
-      # arguments declared using #argument (this is primarily used
-      # by Thor::Group). Tis will leave us with the remaining
-      # positional arguments.
-      to_parse = args
-      to_parse += opts.remaining unless self.class.strict_args_position?(config)
-      thor_args = Thor::Arguments.new(self.class.arguments)
-      thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
-      @args = thor_args.remaining
-    end
-
-    def is_option (options, p)
-      options.each do |o|
-        if !o.aliases.nil?
-          if (o.aliases.is_a? Array and o.aliases.include? p) or (o.aliases.is_a? Array and o.aliases.size ==1 and o.aliases[0].split(",").include? p) or o.switch_name.eql? p
-            return o
-          end
-        end
-
-      end
-      return false
-    end
-
-  end
-end
-
+require 'pry'
 
+# DEV VERSION
+#
 module Cnvrg
-
   class CLI < Thor
 
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
     IP="localhost"
     PORT=7654
+    desc '', ''
+
+    def printable_commands(all = true, subcommand = false)
+      (all ? all_commands : commands).map do |_, command|
+        next if command.hidden? or (command.description.empty? and command.usage.empty?)
+        item = []
+        item << banner(command, false, subcommand)
+        item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
+        item
+      end.compact
+    end
+
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
       def is_thor_reserved_word?(word, type)
@@ -150,10 +58,9 @@ module Cnvrg
         super
       end
     end
-    desc "data", "upload and manage datasets", :hide => true
-    subcommand "data", Data
 
-    desc "", "", :hide => true
+    desc "", ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
 
     def test
       # image_settings = {
@@ -171,10 +78,11 @@ module Cnvrg
       #         },
       #     },
       # }
-      # container = Docker::Container.get('b4d64bf83f41')
-      # s = "/leah/1/2/3/4/5"
-      # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-      # puts container.exec(command, tty: true)
+      container = Docker::Container.get('b4d64bf83f41')
+      s = "/leah/1/2/3/4/5"
+      command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+      puts container.exec(command, tty: true)
+
     end
 
 
@@ -187,7 +95,7 @@ module Cnvrg
 
     map %w(-v --version) => :version
 
-    desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
+    desc 'set api url', 'set api url'
 
     def set_api_url(url)
       home_dir = File.expand_path('~')
@@ -203,19 +111,15 @@ module Cnvrg
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
         owner = config.to_h[:owner]
-        compression_path =  "#{File.expand_path('~')}/.cnvrg/tmp"
 
         say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
         if config.empty?
-          config = {owner: "", username: "", version_last_check: get_start_day(), api: url,compression_path: compression_path }
+          config = {owner: "", username: "", version_last_check: get_start_day(), api: url}
         else
-          if !config.to_h[:compression_path].nil?
-            compression_path = config.to_h[:compression_path]
-          end
-          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
+          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url}
         end
         res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
-        if Cnvrg::CLI.is_response_success(res,false)
+        if Cnvrg::CLI.is_response_success(res)
 
           checks = Helpers.checkmark
 
@@ -233,7 +137,7 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def set_remote_api_url(owner, current_user, url)
       home_dir = File.expand_path('~')
@@ -248,9 +152,7 @@ module Cnvrg
           FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-
-        compression_path =  "#{home_dir}/.cnvrg/tmp"
-        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url,compression_path:compression_path}
+        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url}
         File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
         say "Done"
       rescue
@@ -260,108 +162,57 @@ module Cnvrg
 
     map %w(-api --api) => :set_api_url
 
-    desc 'set_default_owner', 'set default owner'
+    desc 'set default owner', 'set default owner'
 
     def set_default_owner
-      begin
-        path = File.expand_path('~')+"/.cnvrg/config.yml"
-        if !File.exist?(path)
-          say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
-
-          exit(0)
-        end
-        config = YAML.load_file(path)
 
-        username = config.to_h[:username]
-        res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
-        if Cnvrg::CLI.is_response_success(res)
-          owner = username
-          result = res["result"]
-          owners = result["owners"]
-          urls = result["urls"]
-          choose_owner = result["username"]
-          if owners.empty?
-          else
-            owners << choose_owner
-            chosen = false
-            while !chosen
-              owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
-              choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
-
-              if choose_owner =~ /[[:digit:]]/
-                ow_index = choose_owner.to_i-1
-                if ow_index<0 or ow_index >= owners.size
-                  say "No such owner, please choose again", Thor::Shell::Color::BLUE
-                  chosen = false
-                  next
-                end
-                choose_owner = owners[choose_owner.to_i-1]
-                chosen = true
+      path = File.expand_path('~')+"/.cnvrg/config.yml"
+      if !File.exist?(path)
+        say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
 
-              else
+        exit(0)
+      end
+      config = YAML.load_file(path)
 
-                owners_lower = owners.map { |o| o.downcase }
-                ow_index = owners_lower.index(choose_owner.downcase)
-                if ow_index.nil?
-                  say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-                else
-                  chosen = true
-                end
-              end
+      username = config.to_h[:username]
+      res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
+      if Cnvrg::CLI.is_response_success(res)
+        owner = username
+        result = res["result"]
+        owners = result["owners"]
+        urls = result["urls"]
+        choose_owner = result["username"]
 
+        if owners.empty?
+        else
+          owners << choose_owner
+          chosen = false
+          while !chosen
+            choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
+
+            owners_lower = owners.map { |o| o.downcase }
+            ow_index = owners_lower.index(choose_owner.downcase)
+            if ow_index.nil?
+              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+            else
+              chosen = true
             end
-
-
-          end
-          if set_owner(choose_owner, result["username"], urls[ow_index])
-            say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
-          else
-            say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
           end
-        end
-      rescue SignalException
-        say "\nAborting"
-        exit(1)
-      end
-    end
 
-    desc 'set_compression_path', 'set compression path'
-    method_option :reset, :type => :boolean, :aliases => ["-r","--reset"], :default => false
 
-    def set_compression_path(*compression_path)
-      begin
-        if (compression_path.nil? or compression_path.empty?) and options["reset"]
-          compression_path =  ["#{File.expand_path('~')}/.cnvrg/tmp"]
         end
-        compression_path = compression_path.join(" ")
-        if !Dir.exist? compression_path
-          say "Couldn't find #{compression_path}, please make sure it exist", Thor::Shell::Color::RED
-          exit(0)
-        end
-
-        home_dir = File.expand_path('~')
-        path = "#{home_dir}/.cnvrg/config.yml"
-        if !File.exist?(path)
-          say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
-
-          exit(0)
+        if set_owner(owner, username,urls[ow_index])
+          say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
+        else
+          say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
         end
-        config = YAML.load_file(path)
-        config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
-                  version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
-        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config_new.to_yaml }
-        checks = Helpers.checkmark
-        say "#{checks} Done", Thor::Shell::Color::GREEN
-
-      rescue SignalException
-        say "\nAborting"
-        exit(1)
       end
     end
 
+    map %w(--set-default-owner) => :set_default_owner
 
 
-    desc 'login', 'Authenticate with cnvrg.io platform'
+    desc 'login', 'Authenticate with cnvrg.io and store credentials'
 
     def login
       begin
@@ -389,46 +240,25 @@ module Cnvrg
           owners = result["owners"]
           urls = result["urls"]
           choose_owner = result["username"]
-          ow_index = 0
+
           if owners.empty?
-            choose_owner = result["username"]
           else
-            choose_owner = owners[0]
-          end
+            owners << choose_owner
+            chosen = false
+            while !chosen
+              choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
 
-          # if owners.empty?
-          # else
-          #   owners << choose_owner
-          #   chosen = false
-          #   while !chosen
-          #     owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
-          #     choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
-          #
-          #     if choose_owner =~ /[[:digit:]]/
-          #       ow_index = choose_owner.to_i-1
-          #       if ow_index<0 or ow_index >= owners.size
-          #         say "No such owner, please choose again", Thor::Shell::Color::BLUE
-          #         chosen = false
-          #         next
-          #       end
-          #       choose_owner = owners[choose_owner.to_i-1]
-          #       chosen = true
-          #
-          #     else
-          #
-          #       owners_lower = owners.map { |o| o.downcase }
-          #       ow_index = owners_lower.index(choose_owner.downcase)
-          #       if ow_index.nil?
-          #         say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-          #       else
-          #         chosen = true
-          #       end
-          #     end
-          #
-          #   end
+              owners_lower = owners.map { |o| o.downcase }
+              ow_index = owners_lower.index(choose_owner.downcase)
+              if ow_index.nil?
+                say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+              else
+                chosen = true
+              end
+            end
 
 
-          # end
+          end
 
           if set_owner(choose_owner, result["username"], urls[ow_index])
             say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
@@ -444,11 +274,6 @@ module Cnvrg
 
           exit(1)
         end
-      rescue
-
-        say "Error Occurred, aborting", Thor::Shell::Color::RED
-        logout()
-        exit(1)
       rescue SignalException
 
         say "\nAborting"
@@ -478,6 +303,7 @@ module Cnvrg
 
     def me()
       begin
+
         verify_logged_in(false)
         log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
@@ -488,7 +314,6 @@ module Cnvrg
         end
 
         log_end(0)
-
       rescue SignalException
         log_end(-1)
 
@@ -499,8 +324,8 @@ module Cnvrg
 
     ## Projects
     desc 'new', 'Create a new cnvrg project'
-    method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
-    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
+    method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
+    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
 
     def new(project_name)
       begin
@@ -568,8 +393,8 @@ module Cnvrg
         say "created project successfully", Thor::Shell::Color::GREEN
         say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurred, aborting", Thor::Shell::Color::RED
+        puts e
+        log_end(-1)
         if Dir.exist? working_dir
 
           @project.revert(working_dir)
@@ -587,7 +412,6 @@ module Cnvrg
         exit(1)
       end
     end
-
     desc 'set_image', 'set_image for a project'
 
     def set_image(docker_image)
@@ -624,10 +448,9 @@ module Cnvrg
       @image.update_image_activity(nil, nil)
 
     end
-
-    desc 'link', 'Link current directory to a new cnvrg project'
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
+    desc 'link', 'Link current directory to a cnvrg project'
+    method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
+    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
 
     def link
       begin
@@ -651,7 +474,7 @@ module Cnvrg
         end
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Project.link(owner, project_name, docker)
+        if Project.link(owner,project_name, docker)
           path = Dir.pwd
           @project = Project.new(path)
           @project.generate_idx()
@@ -708,834 +531,143 @@ module Cnvrg
         exit(1)
       end
     end
-
-    desc 'data init', 'Init dataset directory'
-    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
-
-    def init_data(public)
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-        dataset_name =File.basename(Dir.getwd)
-        if File.directory?(Dir.getwd+"/.cnvrg")
-          config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
-          say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
-
-          exit(0)
-        end
-        say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
-
-        working_dir = Dir.getwd
-        owner = CLI.get_owner
-        if Dataset.init(owner, dataset_name, options["public"])
-          path = Dir.pwd
-          @dataset = Dataset.new(path)
-
-          url = @dataset.url
-          check = Helpers.checkmark
-          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
-          say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
-          log_end(0)
-
-        else
-          log_end(1, "can't create dataset")
-          @dataset.revert(working_dir) unless @dataset.nil?
-          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
-          exit(0)
-        end
-      rescue SignalException
-        log_end(-1)
-
-        say "\nAborting"
-        exit(1)
-      end
+    desc 'unlink','Unlink a porject'
+    def unlink
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir()
+      list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
+      FileUtils.rm_rf list_to_del
     end
 
-    desc 'data clone', 'Clone dataset'
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
+    desc 'clone', 'Clone a project'
+    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
 
-    def clone_data(dataset_url)
+    def clone(project_url)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        url_parts = dataset_url.split("/")
-        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
+        url_parts = project_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
         slug = url_parts[project_index+1]
         owner = url_parts[project_index-1]
-        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
-
+        remote = options["remote"] || false
+        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
         Cnvrg::CLI.is_response_success(response)
-        dataset_name = response["result"]["name"]
-        if (Dir.exists? dataset_name)
-          say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
-          if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-            say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
-            log_end(1, "conflict with dir #{dataset_name}")
+        response = JSON.parse response["result"]
+        project_name = response["title"]
+        commit_to_clone = options["commit"] || nil
+
+        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
+        clone_resp = false
+        if remote
+          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
+          project_home = Dir.pwd
+        else
+          if (Dir.exists? project_name)
+            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
+            if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
+              log_end(1, "conflict with dir #{project_name}")
+
+              exit(1)
+            end
 
-            exit(1)
           end
+          clone_resp = Project.clone_dir(slug, owner, project_name)
+          project_home = Dir.pwd+"/"+project_name
+
 
         end
-        if Dataset.clone(owner, dataset_name, slug)
-          say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
 
-          commit_to_clone = options["commit"] || nil
-          working_dir = "#{Dir.pwd}/#{dataset_name}"
-          @dataset = Dataset.new(working_dir)
-          @dataset.generate_idx()
-          say "Downloading data", Thor::Shell::Color::BLUE
+        if clone_resp
+          @project = Project.new(project_home)
+          @files = Cnvrg::Files.new(@project.owner, slug)
+          response = @project.clone(remote, commit_to_clone)
+          Cnvrg::CLI.is_response_success response
+          working_dir = project_home
+          docker_image = response["result"]["image"]
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          if !docker_image.nil? and !docker_image.empty? and !remote
+            local_images = Docker::Image.all
+            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
+            if docker_image_local.size == 0
+              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+                image = pull(docker_image)
+                if image
+                  say "downloaded image: #{docker_image}"
+                  @image = Images.new(working_dir, docker_image)
+                else
+                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  @project.revert(working_dir)
+                  exit(1)
+                end
+              end
 
+            elsif docker_image_local.size == 1
+              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, docker_image_local[0])
+            elsif docker_image_local.size >1
+              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, image_name)
+            end
 
-          download_data(false, false, path = working_dir, in_dir=false)
+          end
 
+          successful_changes = []
+          say "Downloading files", Thor::Shell::Color::BLUE
+          if !response["result"]["tree"].nil?
+            response["result"]["tree"].each do |f|
+              relative_path = f[0].gsub(/^#{@project.local_path}/, "")
+              if f[0].end_with? "/"
+                # dir
+                if @files.download_dir(f[0], relative_path, project_home)
+                  successful_changes << relative_path
+                end
+              else
+                # blob
+                if @files.download_file_s3(f[0], relative_path, project_home)
+                  successful_changes << relative_path
+                end
+              end
+            end
+          end
 
-          check = Helpers.checkmark
-          say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
+          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
           log_end(0)
-
         else
-          log_end(1, "can't create dataset")
-          @dataset.revert(working_dir) unless @dataset.nil?
-          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
-          exit(0)
+          log_end(1, "can't create directory")
+
+          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
+          exit(1)
         end
       rescue SignalException
         log_end(-1)
-
         say "\nAborting"
         exit(1)
       end
+
     end
 
-    desc 'init_data_container', 'Init dataset directory', :hide => true
-    method_option :login_content, :type => :string, :aliases => ["-l"], :default => ""
 
-    def init_data_container(container)
+    desc 'status', 'Show the working tree status'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+
+
+    def status
       begin
-        login_content = options["login_content"]
-
-        container = Docker::Container.get(container)
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-
-      rescue SignalException
-        log_end(-1)
-
-        say "\nAborting"
-        exit(1)
-      end
-    end
-
-    desc 'data_snap', 'Init dataset directory', :hide => true
-    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
-
-    def snap_data
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-
-        owner = CLI.get_owner
-        path = Dir.pwd
-        @dataset = Dataset.new(path)
-
-        log_end(0)
-
-      rescue SignalException
-        log_end(-1)
-
-        say "\nAborting"
-        exit(1)
-      end
-    end
-
-    desc 'data_snap', 'Init dataset directory', :hide => true
-
-    def data_init_container(owner, dataset_slug, dataset_name)
-
-      if Dataset.init_container(owner, dataset_slug, dataset_name)
-
-        say "init finished successfully", Thor::Shell::Color::GREEN
-        log_end(0)
-
-      else
-        log_end(1, "can't create dataset")
-        say "error creating dataset, please contact support.", Thor::Shell::Color::RED
-        exit(0)
-      end
-    end
-
-    desc 'data download', 'pull data'
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-
-    def download_data(verbose, sync, path=Dir.pwd, in_dir=true)
-      begin
-        verify_logged_in(in_dir)
-        log_start(__method__, args, options)
-        if path.nil? or path.empty?
-          path = Dir.pwd
-        end
-        dataset_dir = is_cnvrg_dir(path)
-        @dataset = Dataset.new(dataset_dir)
-
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        new_branch = options["new_branch"] || false
-
-        res = @dataset.compare_idx(new_branch)["result"]
-
-        result = res["tree"]
-
-        commit = res["commit"]
-        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
-          log_end(0)
-          return true
-        end
-        result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
-
-        delete = result["result"]["delete"]
-        commits = result["result"]["commits"]
-        updated_idx = result["result"]["idx"]
-        commits.each do |c|
-          file_name = @files.download_data_file(c, dataset_dir)
-
-          if file_name.eql? false or file_name.nil?
-            ##RETRY
-            next
-          end
-          file_path = "#{dataset_dir}/#{file_name}"
-          success = extarct_tar(file_path, dataset_dir)
-          if !success
-            #TODO: ?
-          end
-
-          FileUtils.rm_rf([file_path])
-
-        end
-        to_delete = []
-        delete.each do |d|
-          to_delete << "#{dataset_dir}/#{d}"
-        end
-        FileUtils.rm_rf(to_delete)
-
-        @dataset.update_idx(updated_idx)
-
-
-        # result["conflicts"].each do |f|
-        #   relative_path = f.gsub(/^#{@dataset.local_path}/, "")
-        #   if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
-        #     successful_changes << relative_path
-        #   end
-        #
-        # end
-        # update idx with latest commit
-        # @dataset.update_idx_with_commit!(commit)
-        check = Helpers.checkmark()
-        say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
-
-
-        log_end(0)
-      end
-    rescue
-      log_end(-1)
-
-      say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
-      exit(1)
-    rescue SignalException
-      log_end(-1)
-      say "\nAborting", Thor::Shell::Color::BLUE
-      exit(1)
-    end
-
-
-    desc 'upload_data', 'Upload data files', :hide => true
-    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-
-    def upload_data(sync=false, direct=false)
-
-      begin
-        verify_logged_in(true)
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        ignore = options[:ignore] || []
-        if !@dataset.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
-        end
-        result = @dataset.compare_idx(false)
-
-        commit = result["result"]["commit"]
-        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-          log_end(0)
-
-          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
-          exit(1)
-        end
-
-        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
-        result = result["result"]["tree"]
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "conflict"
-        #   else
-        #     num =  "conflicts"
-        #   end
-        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #
-        # end
-        check = Helpers.checkmark()
-
-        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_end(0)
-          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
-          return true
-        end
-        update_count = 0
-        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
-        successful_updates = []
-        successful_deletions = []
-        if options["verbose"]
-          if update_total == 1
-            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
-          else
-            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
-          end
-        else
-          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
-
-        end
-
-        # Start commit
-
-        commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
-        # upload / update
-        begin
-          (result["added"] + result["updated_on_local"]).each do |f|
-            absolute_path = "#{@dataset.local_path}/#{f}"
-            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-            if File.directory?(absolute_path)
-              resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-              if resDir
-                update_count += 1
-                successful_updates<< relative_path
-              end
-            else
-              res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-
-              if res
-                update_count += 1
-                successful_updates<< relative_path
-              else
-                @files.rollback_commit(commit_sha1)
-                log_end(1, "can't upload, Rolling Back all changes")
-                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-                exit(0)
-              end
-            end
-          end
-
-          # delete
-          deleted = update_deleted(result["deleted"])
-          deleted.each do |f|
-            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-            if relative_path.end_with?("/")
-              if @files.delete_dir(f, relative_path, commit_sha1)
-                # update_count += 1
-                successful_updates<< relative_path
-              end
-            else
-              if @files.delete_file(f, relative_path, commit_sha1)
-                # update_count += 1
-                successful_updates<< relative_path
-              end
-            end
-          end
-
-        rescue SignalException
-          log_end(-1)
-          @files.rollback_commit(commit_sha1)
-          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
-          exit(0)
-        rescue => e
-          log_end(-1, e.message)
-          @files.rollback_commit(commit_sha1)
-          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
-          exit(0)
-        end
-        if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count += result["deleted"].size
-        end
-        if update_count == update_total
-          res = @files.end_commit(commit_sha1)
-          if (Cnvrg::CLI.is_response_success(res, false))
-            # save idx
-            begin
-              @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
-
-              @dataset.update_idx_with_commit!(commit_sha1)
-            rescue => e
-              log_end(-1, e.message)
-              @files.rollback_commit(commit_sha1)
-              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
-              exit(1)
-
-            end
-            if options["verbose"]
-              say "#{check} Done", Thor::Shell::Color::BLUE
-              if successful_updates.size >0
-                say "Updated:", Thor::Shell::Color::GREEN
-                suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say suc.join("\n"), Thor::Shell::Color::GREEN
-              end
-              if successful_deletions.size >0
-                say "Deleted:", Thor::Shell::Color::GREEN
-                del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-                say del.join("\n"), Thor::Shell::Color::GREEN
-              end
-              say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
-            else
-              if (options["sync"] or sync) and direct
-                say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
-
-              else
-                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
-
-              end
-
-            end
-
-            log_end(0)
-          else
-            @files.rollback_commit(commit_sha1)
-            log_end(1, "error. Rolling Back all changes")
-            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
-          end
-        else
-          log_end(1, "error. Rolling Back all changes")
-          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
-
-          @files.rollback_commit(commit_sha1)
-        end
-      rescue => e
-
-        log_end(-1)
-
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
-        @files.rollback_commit(commit_sha1)
-        exit(1)
-      rescue SignalException
-        log_end(-1)
-
-        say "\nAborting", Thor::Shell::Color::BLUE
-        say "\nRolling back all changes", Thor::Shell::Color::BLUE
-        @files.rollback_commit(commit_sha1)
-        exit(1)
-      end
-
-    end
-
-    desc 'data upload', 'push data'
-    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
-
-    def upload_data_tar(ignore, verbose, sync,no_compression)
-
-      begin
-        verify_logged_in(true)
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-
-        @dataset = Dataset.new(dataset_dir)
-
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        if !@dataset.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED
-          exit(1)
-        end
-        say "Checking dataset", Thor::Shell::Color::BLUE
-        local_idx = @dataset.generate_idx
-        result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
-
-
-        commit = result["result"]["commit"]
-        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-          log_end(0)
-
-          say "Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::YELLOW
-          exit(1)
-        end
-
-        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
-        result = result["result"]["tree"]
-        check = Helpers.checkmark()
-
-        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
-          log_end(0)
-          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
-          return true
-        end
-        update_count = 0
-        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
-        successful_updates = []
-        successful_deletions = []
-
-        # Start commit
-        res = @files.start_commit(false)["result"]
-        commit_sha1 =res["commit_sha1"]
-        commit_time = res["commit_time"]
-        # upload / update
-        begin
-          (result["added"] + result["updated_on_local"]).each do |f|
-            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-            successful_updates<< relative_path
-            update_count += 1
-          end
-
-          # delete
-          deleted = update_deleted(result["deleted"])
-          deleted.each do |f|
-            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
-            successful_updates<< relative_path
-          end
-          @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
-
-          @dataset.update_idx_with_commit!(commit_sha1)
-          say "Compressing data", Thor::Shell::Color::BLUE
-
-          home_dir = File.expand_path('~')
-          compression_path = get_compression_path
-          tar_path = "#{compression_path}#{@dataset.slug}_#{commit_sha1}.tar.gz"
-          tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
-          tar_files = (result["added"] + result["updated_on_local"]).join("\n")
-          File.open(tar_files_path, 'w') { |f| f.write tar_files }
-          is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
-          if !is_tar
-            say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
-            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
-            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
-            @files.rollback_commit(commit_sha1)
-            say "Rolling Back all changes.", Thor::Shell::Color::RED
-            exit(1)
-          end
-          say "Uploading data", Thor::Shell::Color::BLUE
-          res = false
-            res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
-
-          if res
-            say "Commiting data", Thor::Shell::Color::BLUE
-
-            cur_idx = @dataset.get_idx.to_h
-
-            res = @files.end_commit_tar(commit_sha1, cur_idx)
-            if !Cnvrg::CLI.is_response_success(res, false)
-              FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
-              FileUtils.rm_rf([tar_path]) if File.exist? tar_path
-
-
-              @files.rollback_commit(commit_sha1)
-              say "Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED
-              exit(1)
-            end
-
-          else
-            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
-            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
-
-
-            @files.rollback_commit(commit_sha1)
-            say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-            exit(1)
-          end
-
-
-          # delete
-          FileUtils.rm_rf([tar_path, tar_files_path])
-
-        rescue SignalException
-          log_end(-1)
-          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
-          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
-
-
-          @files.rollback_commit(commit_sha1)
-          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
-          exit(0)
-        rescue => e
-          puts e.message
-          if !Cnvrg::Helpers.internet_connection?
-            say "Seems there is no internet connection", Thor::Shell::Color::RED
-
-          end
-          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
-          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
-
-          log_end(-1, e.message)
-          @files.rollback_commit(commit_sha1)
-          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
-          exit(0)
-        end
-        if verbose
-          say "#{check} Done", Thor::Shell::Color::BLUE
-          if successful_updates.size >0
-            say "Updated:", Thor::Shell::Color::GREEN
-            suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-            say suc.join("\n"), Thor::Shell::Color::GREEN
-          end
-          if successful_deletions.size >0
-            say "Deleted:", Thor::Shell::Color::GREEN
-            del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
-            say del.join("\n"), Thor::Shell::Color::GREEN
-          end
-
-
-          say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
-        else
-          say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
-
-        end
-
-
-      rescue => e
-        log_end(-1, e.message)
-
-
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
-        @files.rollback_commit(commit_sha1)
-        exit(1)
-      rescue SignalException
-        log_end(-1)
-
-        say "\nAborting", Thor::Shell::Color::BLUE
-        say "\nRolling back all changes", Thor::Shell::Color::BLUE
-        @files.rollback_commit(commit_sha1)
-        exit(1)
-      end
-
-      log_end(0)
-
-    end
-
-
-    desc 'unlink', 'Unlink a project from current directory', :hide => true
-
-    def create_volume
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      @dataset.create_volume()
-
-    end
-
-    desc 'data list', 'List all dataset you currently have'
-
-    def list_dataset
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      owner = @dataset.owner
-      if owner.nil? or owner.empty?
-        owner = CLI.get_owner()
-      end
-      result = @dataset.list(owner)
-      list = result["result"]["list"]
-
-      print_table(list)
-
-    end
-
-    desc 'data commits', 'List all commits for a specific dataset'
-
-    def list_dataset_commits()
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      result = @dataset.list_commits()
-      list = result["result"]["list"]
-
-      print_table(list)
-
-    end
-
-    desc 'commits', 'List all commits for a specific dataset'
-
-    def list_commits()
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-
-      project_dir = is_cnvrg_dir(Dir.pwd)
-      @project = Project.new(project_dir)
-      result = @project.list_commits()
-      list = result["result"]["list"]
-
-      print_table(list)
-
-    end
-
-
-    desc 'unlink', 'Unlink a project from current directory'
-
-    def unlink
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      working_dir = is_cnvrg_dir()
-      list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
-      FileUtils.rm_rf list_to_del
-    end
-
-    desc 'clone', 'Clone project'
-    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
-    method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
-
-    def clone(project_url)
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-        url_parts = project_url.split("/")
-        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
-        slug = url_parts[project_index+1]
-        owner = url_parts[project_index-1]
-        remote = options["remote"] || false
-        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
-        Cnvrg::CLI.is_response_success(response)
-        response = JSON.parse response["result"]
-        project_name = response["title"]
-        commit_to_clone = options["commit"] || nil
-
-        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
-        clone_resp = false
-        if remote
-          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
-          project_home = Dir.pwd
-        else
-          if (Dir.exists? project_name)
-            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
-            if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
-              log_end(1, "conflict with dir #{project_name}")
-
-              exit(1)
-            end
-
-          end
-          clone_resp = Project.clone_dir(slug, owner, project_name)
-          project_home = Dir.pwd+"/"+project_name
-
-
-        end
-
-        if clone_resp
-          @project = Project.new(project_home)
-          @files = Cnvrg::Files.new(@project.owner, slug)
-          response = @project.clone(remote, commit_to_clone)
-          Cnvrg::CLI.is_response_success response
-          working_dir = project_home
-          docker_image = response["result"]["image"]
-          current_commit = response["result"]["commit"]
-          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
-          if !docker_image.nil? and !docker_image.empty? and !remote
-            local_images = Docker::Image.all
-            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
-            if docker_image_local.size == 0
-              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-                image = pull(docker_image)
-                if image
-                  say "downloaded image: #{docker_image}"
-                  @image = Images.new(working_dir, docker_image)
-                else
-                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
-                  @project.revert(working_dir)
-                  exit(1)
-                end
-              end
-
-            elsif docker_image_local.size == 1
-              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, docker_image_local[0])
-            elsif docker_image_local.size >1
-              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
-              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, image_name)
-            end
-
-          end
-
-          successful_changes = []
-          say "Downloading files", Thor::Shell::Color::BLUE
-          if !response["result"]["tree"].nil?
-            response["result"]["tree"].each do |f|
-              relative_path = f[0].gsub(/^#{@project.local_path}/, "")
-              if f[0].end_with? "/"
-                # dir
-                if @files.download_dir(f[0], relative_path, project_home)
-                  successful_changes << relative_path
-                end
-              else
-                # blob
-
-                if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
-                  successful_changes << relative_path
-                end
-              end
-            end
-          end
-
-          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
-          log_end(0)
-        else
-          log_end(1, "can't create directory")
-
-          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
-          exit(1)
-        end
-      rescue SignalException
-        log_end(-1)
-        say "\nAborting"
-        exit(1)
-      end
-
-    end
-
-
-    desc 'status', 'Show the working tree status'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-
-
-    def status
-      begin
-        verify_logged_in()
-        log_start(__method__, args, options)
-        @project = Project.new(get_project_home)
-        new_branch = options["new_branch"] || false
+        verify_logged_in()
+        log_start(__method__, args, options)
+        @project = Project.new(get_project_home)
+        new_branch = options["new_branch"] || false
 
         result = @project.compare_idx(new_branch)["result"]
         commit = result["commit"]
         result = result["tree"]
         say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
-
         if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
           say "Project is up to date", Thor::Shell::Color::GREEN
           log_end(0)
@@ -1582,36 +714,14 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
-
-    def revert_exp
-      begin
-        log_start(__method__, args, options)
-        @project = Project.new(get_project_home)
-
-        result = @project.compare_idx(false)["result"]
-        result = result["tree"]
-        if result["added"].size > 0
-          FileUtils.rm_rf(result["added"])
-        end
-        say "Changes were removed successfully", Thor::Shell::Color::GREEN
-
-
-      rescue SignalException
-        log_end(-1)
-        say "\nAborting"
-        exit(1)
-      end
-    end
-
 
     desc 'upload', 'Upload updated files'
-    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
 
-    def upload(link=false, sync=false, direct=false, ignore_list="")
+    def upload(link=false, sync=false)
 
       begin
         verify_logged_in(true)
@@ -1620,30 +730,12 @@ module Cnvrg
         @project = Project.new(get_project_home)
 
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        ignore = options[:ignore] || ""
-
-        if ignore.nil? or ignore.empty?
-          ignore = ignore_list
-        end
-        data_ignore = data_dir_include()
-        if !data_ignore.nil?
-          if ignore.nil? or ignore.empty?
-            ignore = data_ignore
-          else
-            ignore ="#{ignore},#{data_ignore}"
-          end
-        end
+        ignore = options[:ignore] || []
         if !@project.update_ignore_list(ignore)
           say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
         end
         new_branch = options["new_branch"] || false
 
-        if options["sync"] or sync
-          new_branch_exp = @project.get_new_branch
-          if new_branch_exp
-            new_branch = new_branch_exp
-          end
-        end
         result = @project.compare_idx(new_branch)
         commit = result["result"]["commit"]
         if !link
@@ -1657,23 +749,9 @@ module Cnvrg
           say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
         end
         result = result["result"]["tree"]
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "conflict"
-        #   else
-        #     num =  "conflicts"
-        #   end
-        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #
-        # end
-        check = Helpers.checkmark()
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_end(0)
-          say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
+          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
           return true
         end
         update_count = 0
@@ -1687,7 +765,7 @@ module Cnvrg
             say "Updating #{update_total} files", Thor::Shell::Color::BLUE
           end
         else
-          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
+          say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
 
         end
 
@@ -1721,21 +799,21 @@ module Cnvrg
           end
 
           # delete
-          deleted = update_deleted(result["deleted"])
-          deleted.each do |f|
+          result["deleted"].each do |f|
             relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
             if relative_path.end_with?("/")
               if @files.delete_dir(f, relative_path, commit_sha1)
-                # update_count += 1
+                update_count += 1
                 successful_updates<< relative_path
               end
             else
               if @files.delete_file(f, relative_path, commit_sha1)
-                # update_count += 1
+                update_count += 1
                 successful_updates<< relative_path
               end
             end
           end
+          log_end(0)
 
         rescue SignalException
           log_end(-1)
@@ -1743,14 +821,11 @@ module Cnvrg
           say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
           exit(0)
         rescue => e
-          log_end(-1, e.message)
+          log_end(1, e.message)
           @files.rollback_commit(commit_sha1)
           say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
           exit(0)
         end
-        if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count += result["deleted"].size
-        end
         if update_count == update_total
           res = @files.end_commit(commit_sha1)
           if (Cnvrg::CLI.is_response_success(res, false))
@@ -1760,7 +835,7 @@ module Cnvrg
 
               @project.update_idx_with_commit!(commit_sha1)
             rescue => e
-              log_end(-1, e.message)
+              log_end(1, e.message)
               @files.rollback_commit(commit_sha1)
               say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
               exit(1)
@@ -1770,6 +845,7 @@ module Cnvrg
             if image and image.is_docker
               image.update_image_activity(commit_sha1, nil)
             end
+            check = Helpers.checkmark()
 
             if options["verbose"]
               say "#{check} Done", Thor::Shell::Color::BLUE
@@ -1785,13 +861,7 @@ module Cnvrg
               end
               say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
             else
-              if (options["sync"] or sync) and direct
-                say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
-
-              else
-                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
-
-              end
+              say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN unless options["sync"]
 
             end
 
@@ -1803,14 +873,13 @@ module Cnvrg
           end
         else
           log_end(1, "error. Rolling Back all changes")
-          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
 
           @files.rollback_commit(commit_sha1)
         end
-      rescue => e
+      rescue
         log_end(-1)
 
-        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
         @files.rollback_commit(commit_sha1)
         exit(1)
       rescue SignalException
@@ -1825,57 +894,27 @@ module Cnvrg
     end
 
     desc 'download', 'Download updated files'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
-    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
 
-    def download(sync=false, ignore_list="")
+    def download
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
         project_home = get_project_home
-        @project = Project.new(project_home)
-        @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        ignore = options[:ignore] || ""
-        if ignore.nil? or ignore.empty?
-          ignore = ignore_list
-        end
-        data_ignore = data_dir_include()
-        if !data_ignore.nil?
-          if ignore.nil? or ignore.empty?
-            ignore = data_ignore
-          else
-            ignore ="#{ignore},#{data_ignore}"
-          end
-        end
-        if !@project.update_ignore_list(ignore)
-          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
-        end
+        @project = Project.new(project_home)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
         new_branch = options["new_branch"] || false
 
         res = @project.compare_idx(new_branch)["result"]
-
         result = res["tree"]
-
         commit = res["commit"]
-        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
+        if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
           log_end(0)
           return true
         end
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "1 conflict"
-        #   else
-        #     num =  "#{result["conflicts"].size} conflicts"
-        #   end
-        #   say "Project contains #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix them, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #   end
         update_count = 0
         update_total = result["updated_on_server"].size + result["conflicts"].size
 
@@ -1897,7 +936,7 @@ module Cnvrg
 
         result["conflicts"].each do |f|
           relative_path = f.gsub(/^#{@project.local_path}/, "")
-          if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
+          if @files.download_file_s3(f, relative_path, project_home, conflict=true)
             successful_changes << relative_path
           end
 
@@ -1928,35 +967,12 @@ module Cnvrg
             say successful_changes.join("\n"), Thor::Shell::Color::GREEN
             say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
           else
-            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
+            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
           end
 
 
           log_end(0)
         end
-      rescue => e
-        log_end(-1)
-
-        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
-        if successful_changes.nil?
-          exit(1)
-        end
-        successful_changes.each do |f|
-
-          abs_path = "#{@project.local_path}/#{f}"
-          filename = File.basename abs_path
-          say "revoking #{filename}"
-          if result["conflicts"].include? f
-            @files.revoke_download_file(abs_path, f, filename, true)
-          elsif result["updated_on_server"].include? f
-            if File.directory? abs_path
-              @files.revoke_download_dir(abs_path, f, project_home)
-            else
-              @files.revoke_download_file(project_home, abs_path, filename)
-            end
-          end
-        end
-        exit(1)
       rescue SignalException
         log_end(-1)
         say "\nAborting", Thor::Shell::Color::BLUE
@@ -2010,7 +1026,6 @@ module Cnvrg
         if !response["result"]["tree"].nil?
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
           File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
-          current_tree = Dir.entries(".").reject { |file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" }
 
           response["result"]["tree"].each do |f|
             relative_path = f[0].gsub(/^#{@project.local_path}/, "")
@@ -2022,7 +1037,7 @@ module Cnvrg
               end
             else
               # blob
-              if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=commit_sha1)
+              if @files.download_file_s3(f[0], relative_path, project_home)
                 current_tree.delete(relative_path)
 
                 successful_changes << relative_path
@@ -2040,100 +1055,43 @@ module Cnvrg
     end
 
 
-    desc 'data_jump', 'jump to specific commit', :hide => true
-
-    def data_jump(*commit_sha1)
-      begin
-        verify_logged_in()
-        log_start(__method__, args, options)
-        dataset_dir = is_cnvrg_dir(Dir.pwd)
-        @dataset = Dataset.new(dataset_dir)
-
-        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
-        if commit_sha1.nil? or commit_sha1.empty?
-          commit_sha1 = @dataset.last_local_commit
-        end
-        response = @dataset.compare_commits(commit_sha1)
-        successful_changes = []
-        if !response["result"]["status"].nil?
-          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
-          status = response["result"]["status"]
-          (status["delete"]).each do |f|
-            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
-            FileUtils.rm_rf(relative_path)
-          end
-          # current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
-          (status["dirs"]).each do |f|
-            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
-            # dir
-            if @files.download_dir(dataset_dir, relative_path)
-              # current_tree.delete(relative_path[0, relative_path.size-1])
-              successful_changes << relative_path
-            end
-          end
-          (status["download"]).each do |f|
-            relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
-            # dir
-            if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
-              successful_changes << relative_path
-            end
-          end
-
-
-          say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
-          log_end(0)
-        end
-
-      rescue SignalException
-        log_end(-1)
-        exi(1)
-      end
-    end
-
     desc 'sync', 'Sync with remote server'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
-    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
+    method_option :new_branch, :type => :boolean, :aliases => ["--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
 
-    def sync(direct=true)
-      verify_logged_in(true) if direct
+    def sync
       if options["verbose"]
-        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+      say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
       else
         say 'Syncing project', Thor::Shell::Color::BLUE
       end
 
 
-      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
-      invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
-             :ignore => options[:ignore]
+      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
+      invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
+      check = Helpers.checkmark()
 
+      say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
 
     end
 
 
     desc 'run cmd', 'Runs an experiment'
-    method_option :local, :type => :boolean, :aliases => ["-l", "--local"], :default => false
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
+    method_option :sync_before, :type => :boolean, :aliases => ["--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["--t"], :default => ""
     method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
-    method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
-    method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
-    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
-    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
+    method_option :email_notification, :type => :boolean, :aliases => ["--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
 
     def run(*cmd)
       sync_before = options["sync_before"]
@@ -2146,73 +1104,38 @@ module Cnvrg
       local = options["local"]
       schedule = options["schedule"]
       image = options["image"]
-      grid = options["grid"]
-      data = options["data"]
-      data_commit = options["data_commit"]
-      ignore = options["ignore"]
-      options_hash = Hash[options]
-      real_options = []
-      options_hash.each do |o|
-        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu", "gpuxl", "gpuxxl"].include? o[0])
-      end
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-               :log => log, :email_notification => email_notification, :upload_output => upload_output,
-               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
+               :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
         return
       else
-        real_options.delete(["local", false])
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
         instance_type = get_instance_type(instances)
-        if !instance_type.nil? and !instance_type.empty?
-          real_options << ["machine_type", instance_type]
-        end
-        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
-               :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
+               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image
         return
       end
-
-      # if local
-      #
-      # else
-
-
-      #   invoke :exec_remote, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-      #          :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
-      #          :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
-      #   return
-      # end
     end
 
 
-    desc '', '', :hide => true
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb,--sync_before"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa,--sync_after"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
-    method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en,--email_notification"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["-uo,--upload_output"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
-    method_option :indocker, :type => :boolean, :aliases => ["--indocker"], :default => false
-    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
-    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
-    method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    def exec(*cmd)
+    desc '', ''
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
+    method_option :indocker, :type => :boolean,  :default => false
 
+    def exec(*cmd)
       log = []
       cpu_average =0
       memory_average = 0
       verify_logged_in(true)
       log_start(__method__, args, options)
-      working_dir = is_cnvrg_dir
-      script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
+      working_dir = Dir.pwd
 
 
       sync_before = options["sync_before"]
@@ -2220,14 +1143,11 @@ module Cnvrg
       print_log = options["log"]
       title = options["title"]
       commit = options["commit"] || nil
-      image = options["image"] || nil
-      indocker = options["indocker"] || false
-      ignore = options[:ignore] || ""
-
+      image = options["image"]  || nil
+      indocker = options["indocker"]  || false
 
       email_notification = options["email_notification"]
       upload_output = options["upload_output"]
-      upload_output = "1m" if upload_output.nil? or upload_output.empty?
       time_to_upload = calc_output_time(upload_output)
       project_home = get_project_home
       @project = Project.new(project_home)
@@ -2239,8 +1159,7 @@ module Cnvrg
         else
           if sync_before
             # Sync before run
-
-            invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
+            invoke :sync, [], :new_branch => is_new_branch
           end
         end
         #set image for the project
@@ -2248,46 +1167,40 @@ module Cnvrg
           invoke :set_image, [image]
         end
         if !indocker
-          image_proj = is_project_with_docker(working_dir)
+        image_proj = is_project_with_docker(working_dir)
 
 
-          if image_proj and image_proj.is_docker
-            container= image_proj.get_container
-            if !container
-              say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
-              exit(1)
-            end
+        if image_proj and image_proj.is_docker
+          container= image_proj.get_container
+          if !container
+            say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
 
 
-            exec_args = args.flatten.join(" ")
-            options_hash = Hash[options]
-            options_hash.except!("image", "indocker")
-            exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-            command_to_run = copy_args.join(" ")
-            command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
-            puts container.exec(command, tty: true)
-            container.stop()
-            exit(0)
-          end
+          exec_args = args.flatten.join(" ")
+          options_hash = Hash[options]
+          options_hash.except!("image", "indocker")
+          exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+          command_to_run = cmd.join(" ")
+          command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
+          puts container.exec(command, tty: true)
+          container.stop()
+          exit(0)
         end
-        remote = options["remote"]
-        if remote
-          docker_id = `cat /etc/hostname`
-          docker_id = docker_id.strip()
         end
-        is_on_gpu = options["gpu"]
         start_commit = @project.last_local_commit
         cmd = cmd.join("\s")
 
+        say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
+
         @exp = Experiment.new(@project.owner, @project.slug)
 
         platform = RUBY_PLATFORM
         machine_name = Socket.gethostname
         begin
-          machine_activity = @exp.get_machine_activity(working_dir)
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
-          say "Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
-          say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
+          machine_activity = @exp.get_machine_activity(Dir.pwd)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -2299,34 +1212,19 @@ module Cnvrg
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
-                    if remote
-                      stats = usage_metrics_in_docker(docker_id)
-                      cpu = stats[0]
-                      memory = stats[1]
-                    else
-                      memory = memory_usage()
-                      cpu = cpu_usage()
-                    end
-
+                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
+                    monitor_by = monitor.split(" ")
+                    memory = monitor_by[2]
+                    cpu = monitor_by[1]
                     memory_total << memory.to_f
                     cpu_total << cpu.to_f
                     real_time= Time.now-real
 
-                    cur_log = { time: cur_time,
+                    cur_log = {time: cur_time,
                                message: line,
                                type: "stdout",
-                               real: real_time,
-                                cpu: cpu,
-                                memory: memory
-                    }
-                    if is_on_gpu
-                      gpu_stats = gpu_util
-                      gpu_utilization = gpu_stats[0]
-                      gpu_memory_util = gpu_stats[1]
-                      cur_log.merge!(gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
-                    end
-
-                      if print_log
+                               real: real_time}
+                    if print_log
                       puts cur_log
                     end
                     log << cur_log
@@ -2335,8 +1233,8 @@ module Cnvrg
                       if time_to_upload !=0
                         if time_to_upload <= Time.now - start_loop
                           #upload current log
-                          # cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
-                          # memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+                          cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                          memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
 
                           @exp.upload_temp_log(log, cpu_average, memory_average)
                           log = []
@@ -2360,7 +1258,8 @@ module Cnvrg
                   end
 
                 rescue Errno::EIO => e
-                  # break
+                  puts e
+                    # break
                 rescue Errno::ENOENT
                   log_end(1, "command #{cmd} isn't valid")
 
@@ -2372,11 +1271,8 @@ module Cnvrg
                   exp_success = false
                   say "The process exited!", Thor::Shell::Color::RED
                 rescue => e
-                  log_end(-1, e.message)
-                  res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
-
-                  say "Error occurred,aborting", Thor::Shell::Color::RED
-                  exit(0)
+                  puts e
+                  log_end(1, e.message)
                 end
                 ::Process.wait pid
                 cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
@@ -2387,7 +1283,7 @@ module Cnvrg
                 end
                 if !exp_success
                   if !Cnvrg::Helpers.internet_connection?
-                    wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
+                    wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
                     if wait_offline
                       say "Waiting until your'e online..", Thor::Shell::Color::BLUE
                       while !Cnvrg::Helpers.internet_connection?
@@ -2410,21 +1306,23 @@ module Cnvrg
                 end
                 if sync_after
                   # Sync after run
+                  if !commit.nil?
+                    invoke :sync, [], :new_branch => true
 
-
-                  download(sync=true, ignore_list=ignore)
-                  upload(link=false, sync=true, direct=false, ignore_list=ignore)
-
+                  else
+                    invoke :sync, [], :new_branch => true
+                  end
                 end
                 end_commit = @project.last_local_commit
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                 check = Helpers.checkmark()
-                say "#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN
+                say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
                 log_end(0)
               end
             rescue => e
-              log_end(-1, e.message)
+              puts e
+              log_end(1, e.message)
               if container
                 container.stop()
               end
@@ -2451,7 +1349,7 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
+    desc '', ''
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
@@ -2490,7 +1388,7 @@ module Cnvrg
         machine_name = Socket.gethostname
         begin
 
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -2560,9 +1458,7 @@ module Cnvrg
                   exp_success = false
                   say "The process exited!", Thor::Shell::Color::RED
                 rescue => e
-                  log_end(-1, e.message)
-                  say "Error occurred, aborting", Thor::Shell::Color::RED
-                  exit(0)
+                  log_end(1, e.message)
                 end
                 ::Process.wait pid
                 cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
@@ -2612,7 +1508,7 @@ module Cnvrg
                 log_end(0)
               end
             rescue => e
-              log_end(-1, e.message)
+              log_end(1, e.message)
               say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
               exit(1)
             end
@@ -2636,33 +1532,23 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
-    method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
-    method_option :machine_type, :type => :string, :default => ""
-    method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
-    method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
-    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
-    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
+    desc '', ''
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => nil
+    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
 
     def exec_remote(*cmd)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
-      path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
-
       begin
-        grid = options["grid"] || nil
-        data = options["data"] || nil
-        data_commit = options["data_commit"] || nil
-        sync_before = options["sync_before"]
 
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
@@ -2684,22 +1570,10 @@ module Cnvrg
           new_time = new_time[0, new_time.size-6] #remove timezone
           schedule = "at #{new_time}"
         end
-        upload_output = options["upload_output"]
-        time_to_upload = calc_output_time(upload_output)
-        if time_to_upload==0 or time_to_upload==-1
-          upload_output_option = "--upload_output=1m"
-        else
-          upload_output_option = "--upload_output=#{upload_output}"
-        end
-        remote = "--remote=true"
-        if !instance_type.nil? and instance_type.include? "gpu"
-          remote= "#{remote} --gpu=true"
-        end
-
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
+        options_hash.except!("schedule", "machine_type", "image")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
+        command = "#{exec_options} --uo=1m #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -2710,20 +1584,19 @@ module Cnvrg
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image, [choose_image]
+          invoke :set_image,[choose_image]
         end
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
           # say "Couldn't find image related to project", Thor::Shell::Color::RED
-
-          image_slug = "cnvrg"
+          default_image_name = "cnvrg"
           if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
-            image_slug = "cnvrg_gpu"
+            default_image_name = "cnvrg-gpu"
           end
           # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
           # if default
-          #   image = Images.new(working_dir, default_image_name)
-          #   image_slug = image.image_slug
+            image = Images.new(working_dir, default_image_name)
+            image_slug = image.image_slug
           # else
           #   exit(0)
           # end
@@ -2731,27 +1604,16 @@ module Cnvrg
           image_slug = image.image_slug
         end
 
-        invoke :sync, [false], []  if sync_before
-
 
-        if command.include? "'"
-          oc = command.to_enum(:scan, /'/).map { Regexp.last_match }
-          pairs = oc.enum_for(:each_slice, 2).to_a
-          pairs.each_with_index do |p,i|
-            add=0
-            if i!=0
-              add=2*i
-            end
-            total_loc = command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add]
-            command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add] = "\"#{total_loc}\""
-          end
 
+        invoke :sync, [], []
 
-        end
         say "Running remote experiment", Thor::Shell::Color::BLUE
         exp = Experiment.new(project.owner, project.slug)
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
+
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
         if Cnvrg::CLI.is_response_success(res)
+
           # if res["result"]["machine"] == -1
           #   say "There are no available machines", Thor::Shell::Color::BLUE
           #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
@@ -2783,13 +1645,7 @@ module Cnvrg
           #   end
           # else
           check = Helpers.checkmark()
-          str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
-
-          if res["result"]["grid"]
-            str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
-          end
-
-          say str, Thor::Shell::Color::GREEN
+          say "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}", Thor::Shell::Color::GREEN
 
           exit(0)
           # end
@@ -2809,30 +1665,21 @@ module Cnvrg
     end
 
     desc 'deploy', 'Deploys model to production'
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :schedule, :type => :string, :aliases => ["--schedule", "-s"], :default => ""
-
-    method_option :commit, :type => :string, :aliases => ["--commit", "-c"], :default => ""
-    method_option :workers, :type => :string, :aliases => ["--workers", "-w"], :default => ""
-    method_option :file_as_input, :type => :boolean, :aliases => ["--input", "-i"], :default => false
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
 
     def deploy(file_to_run, function)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
       begin
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
-        instance_type = get_instance_type(instances)
 
+        instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
 
 
+
         if !schedule.nil? and !schedule.empty?
 
           local_timestamp = get_schedule_date
@@ -2841,34 +1688,26 @@ module Cnvrg
         project = Project.new(working_dir)
         commit_to_run = options["commit"] || nil
 
-        workers = options["workers"] || nil
-        begin
-          num_workers = workers.to_i
-        rescue
-          say "Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED
-          exit(1)
-        end
-        file_as_input = options["file_as_input"] || false
-
 
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
-          # say "Couldn't find image related to project", Thor::Shell::Color::RED
-          # default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
-          # if default
-          image = Images.new(working_dir, "cnvrg")
-          image_slug = image.image_slug
-          # else
-          #   exit(0)
-          # end
+          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
+          if default
+            image = Images.new(working_dir, "cnvrg")
+            image_slug = image.image_slug
+          else
+            exit(0)
+          end
         else
           image_slug = image.image_slug
         end
 
 
-        invoke :sync, [false], []
 
-        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input)
+        invoke :sync, [], []
+
+        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
 
         if Cnvrg::CLI.is_response_success(res)
 
@@ -2924,37 +1763,27 @@ module Cnvrg
 
     method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
-    method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
-    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
-
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
+    desc 'starts a notebook session', 'starts a notebook session remotely or locally'
 
-    desc 'notebook', 'starts a notebook session remotely or locally'
-
-    def notebook
+    def notebook()
       local = options["local"]
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
-      image = options["image"]
-      data = options["data"]
-      data_commit = options["data_commit"]
+
       if local
-        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
+        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel
         return
       else
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
         instance_type = get_instance_type(instances)
 
-        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
-               :data => data, :data_commit => data_commit
+        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type
         return
 
       end
@@ -2963,12 +1792,10 @@ module Cnvrg
     end
 
     desc 'remote_notebook', 'run notebook server on remote server'
-    method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
-    method_option :machine_type, :type => :string, :default => ""
-    method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
-    method_option :image, :type => :string, :aliases => ["-i"], :default => ""
-    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
-    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
 
 
     def remote_notebook()
@@ -2980,15 +1807,13 @@ module Cnvrg
       notebook_dir = options["notebook_dir"]
       instance_type = options["machine_type"] || nil
       kernel = options["kernel"] || nil
-      data = options["data"]
-      data_commit = options["data_commit"]
 
 
       begin
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image, [choose_image]
+          invoke :set_image,[choose_image]
         end
         @image = is_project_with_docker(working_dir)
         if !@image or !@image.is_docker
@@ -2999,16 +1824,17 @@ module Cnvrg
           end
           # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
           # if default
-          @image = Images.new(working_dir, default_image_name)
+            @image = Images.new(working_dir, default_image_name)
           # else
           #   exit(0)
           # end
         end
 
-        invoke :sync, [false], []
+        invoke :sync, [], []
+
 
 
-        res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
+        res = @image.remote_notebook(notebook_dir, instance_type, kernel)
         if Cnvrg::CLI.is_response_success(res)
           if res["result"]["machine"] == -1
             say "There are no available machines", Thor::Shell::Color::BLUE
@@ -3047,7 +1873,7 @@ module Cnvrg
             note_url = res["result"]["notebook_url"]
             @image.set_note_url(note_url)
             check = Helpers.checkmark()
-            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
+            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
             # Launchy.open(url)
 
             exit(0)
@@ -3115,8 +1941,9 @@ module Cnvrg
 
         end
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurred, aborting", Thor::Shell::Color::RED
+        puts e
+        log_end(-1)
+        say "Error occurd, Aborting"
         if container
           container.stop()
         end
@@ -3177,8 +2004,9 @@ module Cnvrg
           end
         end
       rescue => e
-        log_end(-1, e.message)
-        say "Error occurred, aborting"
+        puts e
+        log_end(-1)
+        say "Error occurd, Aborting"
         if container
           container.stop()
         end
@@ -3195,7 +2023,6 @@ module Cnvrg
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i"], :default => ""
 
     def run_notebook
 
@@ -3209,7 +2036,7 @@ module Cnvrg
         notebook_dir = options["notebook_dir"]
         remote = options["remote"] || false
         kernel = options["kernel"] || ""
-        notebooks_pid = nil
+
 
         if notebook_dir.empty?
           notebook_dir = project_dir
@@ -3217,148 +2044,107 @@ module Cnvrg
 
           notebook_dir = project_dir+ notebook_dir
         end
-        choose_image = options["image"]
-
-        if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image, [choose_image]
-        end
-
+        say "Linking notebook directory to: #{File.basename notebook_dir}", Thor::Shell::Color::BLUE
         image = is_project_with_docker(Dir.pwd)
-        if !image
-          jupyter_installed = `which jupyter`
-          if !$?.success?
-            say "Could not find jupyter, Is it installed?", Thor::Shell::Color::RED
-            exit(1)
-          end
-
-
-          cmd = "jupyter-notebook --port=8888"
-          PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
-            begin
-              notebooks_pid = pid
-              stdout.each do |line|
-                puts line
-
-              end
-
-            rescue Errno::EIO => e
-              # break
-            rescue Errno::ENOENT
-              log_end(1, "command #{cmd} isn't valid")
-
-
-              say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
-            rescue PTY::ChildExited
-              log_end(1, "proccess exited")
-              say "The process exited!", Thor::Shell::Color::RED
-            rescue => e
-              log_end(-1, e.message)
-              say "Error occurred,aborting", Thor::Shell::Color::RED
-              exit(0)
-
-            end
-
+        if image and image.is_docker and !remote
+          container= image.get_container
+          if !container
 
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
           end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
 
         end
+        if options["verbose"]
+          say "Syncing project before running", Thor::Shell::Color::BLUE
+          say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+        end
+        @project = Project.new(project_dir)
 
-        if image and image.is_docker and !remote
-          container= image.get_container
-          if !container
-            say "Couldn't start docker container", Thor::Shell::Color::RED
-            exit(1)
+        start_commit = @project.last_local_commit
 
+        if (note_slug=image.note_slug)
+          say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
+          new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
+          if !new
+            exit(0)
           end
 
-          if options["verbose"]
-            say "Syncing project before running", Thor::Shell::Color::BLUE
-            say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
-          end
-          @project = Project.new(project_dir)
+        end
+        invoke :sync, [], :verbose => options["verbose"]
+        say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
+        #replace url
+        base_url = get_base_url()
 
-          start_commit = @project.last_local_commit
+        local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
+        command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+        container.exec(command, tty: true)
+        container.stop()
+        container.start()
 
-          if (note_slug=image.note_slug)
-            say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
-            new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
-            if !new
-              exit(0)
-            end
+        @note = Experiment.new(@project.owner, @project.slug)
+        port = image.container_port()
 
-          end
-          invoke :sync, [false], :verbose => options["verbose"]
-          say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
-          #replace url
-          base_url = get_base_url()
-
-          local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
-          command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-          container.exec(command, tty: true)
-          container.stop()
-          container.start()
-          sleep(7)
-          @note = Experiment.new(@project.owner, @project.slug)
-          port = image.container_port()
-
-          command = ["/bin/bash", "-lc", "jupyter notebook list"]
-          list = container.exec(command, tty: true)[0]
-          if list.empty? or list.nil?
-            say "Couldn't start notebook server", Thor::Shell::Color::RED
-            log_end(1, "can't start notebook server")
-            exit(1)
-          end
+        command = ["/bin/bash", "-lc", "jupyter notebook list"]
+        list = container.exec(command, tty: true)[0]
+        if list.empty? or list.nil?
+          say "Couldn't start notebook server", Thor::Shell::Color::RED
+          log_end(1, "can't start notebook server")
+          exit(1)
+        end
 
-          result = ""
-          list.each do |r|
-            if r.include? "http"
-              result = r
-            end
-          end
-          token = result.to_s.split("::")[0].to_s.match(/(token=)(.+)\s/)[2]
+        url =  list[list.size-1].split(" ::")
+        token = url[0][(url[0].index("=")+1)..-1]
 
-          # machine_activity = @note.get_machine_activity(project_dir)
+        # machine_activity = @note.get_machine_activity(project_dir)
 
 
-          slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
-          image.set_note_url(slug)
-          note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
+        slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
+        image.set_note_url(slug)
+        note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
 
 
-          if !note_url.empty?
-            check = Helpers.checkmark()
+        if !note_url.empty?
+          check = Helpers.checkmark()
 
-            say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
-            Launchy.open(note_url)
-          else
-            say "Couldn't start notebook server", Thor::Shell::Color::RED
-            log_end(1, "can't start notebook server")
-            exit(1)
-          end
+          say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
+          Launchy.open(note_url)
+        else
+          say "Couldn't start notebook server", Thor::Shell::Color::RED
+          log_end(1, "can't start notebook server")
+          exit(1)
         end
 
-
+          # jup =verify_software_installed("jupyter-notebook")
+          # logs = `#{jup} --no-browser --ip=0.0.0.0  --notebook-dir=#{notebook_dir}`
+          # url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
+          # if !url.empty?
+          #   check = Helpers.checkmark()
+          #
+          #   say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
+          #   log_end(0)
+          # else
+          #   say "Couldn't start notebook server", Thor::Shell::Color::RED
+          #   log_end(1, "can't start notebook server")
+          #   exit(1)
+          # end
       rescue => e
-        log_end(-1, e.message)
+        log_end(-1)
+        puts e
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
       rescue SignalException
-        if !notebooks_pid.nil?
-          ::Process.kill(0, notebooks_pid)
-          say "#{check} Notebook has stopped successfully", Thor::Shell::Color::GREEN
-
-
-          invoke :sync, [false], []
-        else
-          log_end(-1)
-
-          if container
-            container.stop()
-          end
+        log_end(-1)
+        if container
+          container.stop()
         end
-
+        say "Aborting"
+        exit(1)
       end
 
 
@@ -3379,7 +2165,7 @@ module Cnvrg
 
 
         say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
-        invoke :sync, [false], :verbose => options["verbose"]
+        invoke :sync, [], :verbose => options["verbose"]
 
         say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
 
@@ -3424,7 +2210,8 @@ module Cnvrg
           exit(0)
         end
       rescue => e
-        log_end(-1, e.message)
+        puts e
+        log_end(-1)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -3474,7 +2261,8 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1, e.message)
+        log_end(-1)
+        puts e
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -3537,12 +2325,14 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1, e.message)
+        log_end(-1)
+        puts e
         say "Error occurd, aborting"
         if container
           container.stop()
         end
       rescue SignalException
+        log_End(-1)
         if container
           container.stop()
         end
@@ -3551,100 +2341,7 @@ module Cnvrg
       end
 
     end
-
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
-    method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
-    method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
-    method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
-
-    desc 'create_custom_image', 'run commands inside containers', :hide=>true
-
-    def create_custom_image(image_name)
-      begin
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                   "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
-      instance_type = get_instance_type(instances)
-      image_extend = options["image"]
-      public = options["public"]
-      base = options["base"]
-      python3 = options["python3"]
-      owner = CLI.get_owner
-      checks = Helpers.checkmark()
-
-      say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
-
-      resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
-      if Cnvrg::CLI.is_response_success(resp,false)
-        image_slug = resp["result"]["slug"]
-        container = resp["result"]["machine_c"]
-        say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
-        say "Connecting to machine", Thor::Shell::Color::BLUE
-        ssh = Ssh.new(resp)
-        if !ssh.is_ssh
-          say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
-          Images.revoke_custom_new_image(owner, image_slug)
-          log_end(-1,"Couldn't connect to machine,aborting")
-        end
-        say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
-        begin
-
-          while true
-            command = ask("$>")
-            puts ssh.exec_command(command)
-          end
-
-        rescue SignalException
-          say "Commiting Image..", Thor::Shell::Color::BLUE
-
-        end
-        resp = Images.commit_custom_image(owner,image_slug)
-        if Cnvrg::CLI.is_response_success(resp,false)
-          say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-
-
-
-
-
-      end
-      rescue =>e
-        puts e
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-      rescue SignalException
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh
-        end
-        say "\nAborting"
-        exit(1)
-      end
-
-    end
-
-
-    desc 'build', 'run commands inside containers'
+    desc 'run commands inside containers', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
     def build(*cmd)
@@ -3655,10 +2352,10 @@ module Cnvrg
         working_dir = is_cnvrg_dir
         install_file = options["install"] || nil
         if !install_file.nil?
-          commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
+          commands = File.open(install_file).read.chop.gsub!("\n",",").split(",")
 
         else
-          commands = [cmd.join(" ")]
+          commands  = [cmd.join(" ")]
         end
 
 
@@ -3676,25 +2373,25 @@ module Cnvrg
 
         end
         commands.each do |c|
-          if c.include? "pip"
-            c.sub("pip", "/opt/ds/bin/pip")
-          end
-          if c.include? "pip3"
-            c.sub("pip3", "/opt/ds3/bin/pip3")
-          end
+        if c.include? "pip"
+          c.sub("pip","/opt/ds/bin/pip")
+        end
+        if c.include? "pip3"
+          c.sub("pip3","/opt/ds3/bin/pip3")
+        end
 
-          say "Running #{c}", Thor::Shell::Color::BLUE
-          command = ["/bin/bash", "-lc", "#{c}"]
-          res = container.exec(command, tty: false)
-          if res[2] != 0
-            say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
-            container.stop()
-            log_end(0)
+        say "Running #{c}", Thor::Shell::Color::BLUE
+        command = ["/bin/bash", "-lc", "#{c}"]
+        res = container.exec(command, tty: false)
+        if res[2] != 0
+         say  "Could not run command: #{c}, #{res[1][0]}" , Thor::Shell::Color::RED
+          container.stop()
+          log_end(0)
 
-            exit(1)
-          end
-          say res[0].join("\n")
-          image.store_image_build_commands(working_dir, c)
+          exit(1)
+        end
+        say res[0].join("\n")
+        image.store_image_build_commands(working_dir,c)
         end
 
         checks = Helpers.checkmark()
@@ -3705,7 +2402,8 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1, e.message)
+        log_end(-1)
+        puts e
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
@@ -3754,9 +2452,8 @@ module Cnvrg
         log_end(0)
         return new_image.id
       rescue => e
-        log_end(-1, e.message)
-        say "\nError occurred,aborting"
-        exit(1)
+        puts e
+
       rescue SignalException
         log_end(-1)
         say "\nAborting"
@@ -3799,7 +2496,7 @@ module Cnvrg
 
     end
 
-    desc 'push', 'push image to cnvrg repository'
+    desc 'push image to cnvrg repository', 'push image to cnvrg repository'
 
     def push(*name)
       verify_logged_in(true)
@@ -3817,24 +2514,22 @@ module Cnvrg
         end
         if !name.nil? and !name.empty?
           if name.include? " "
-            name.gsub!(" ", "_")
+            name.gsub!(" ","_")
           end
         end
-        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
+        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n",",")
         if stored_commands.nil? or stored_commands.empty?
           say "Nothing to push", Thor::Shell::Color::BLUE
           exit(0)
         end
 
         say "Pushing new image", Thor::Shell::Color::BLUE
-        if image.create_custom_image(name, working_dir, stored_commands)
+        if image.create_custom_image(name,working_dir,stored_commands)
 
           say "Image was updated successfully", Thor::Shell::Color::GREEN
         end
       rescue => e
-        log_end(-1, e.message)
-        say "error occurred, aborting"
-
+        puts e
       end
     end
 
@@ -3896,8 +2591,8 @@ module Cnvrg
           exit(1)
         end
       rescue => e
-        log_end(-1, e.message)
-        say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
+        puts e
+
       rescue SignalException
         log_end(-1)
 
@@ -3906,7 +2601,7 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def upload_log()
       log_path = '/home/ds/app/uwsgi.log'
@@ -3915,32 +2610,28 @@ module Cnvrg
 
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def exec_container(container_id, *cmd)
       container = Docker::Container.get(container_id)
       container.start()
       cnvrg_command = cmd.join(" ")
       command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
-      res = container.exec(command, tty: true, wait: 5400)[0]
+      res = container.exec(command, tty: true,wait:5400)[0]
       say res
+
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def port_container(container_id)
       container = Docker::Container.get(container_id)
       say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
-    end
 
-    desc '', '', :hide => true
 
-    def tensor_port_container(container_id)
-      container = Docker::Container.get(container_id)
-      say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def stop_container(container_id)
       container = Docker::Container.get(container_id)
@@ -3949,13 +2640,13 @@ module Cnvrg
 
     end
 
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c"], :default => "/usr/local/cnvrg/run_ipython.sh"
+    desc '', ''
+    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
+    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
+    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
 
 
-    def config_remote(image_name, port=7654, tensport=6006)
+    def config_remote(image_name, port=7654)
       local_images = Docker::Image.all
 
       docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -3968,25 +2659,20 @@ module Cnvrg
         login_content = options["login"]
         app_dir = options["app_dir"]
         cmd= options["cmd"]
-        volume_from = options["volume"]
 
         image_settings = {
             'Image' => "#{image_name}:latest",
-
+            'User' => 'ds',
             'Cmd' => cmd,
             'WorkingDir' => app_dir,
             'ExposedPorts' => {
                 '8888/tcp' => {},
             },
             'HostConfig' => {
-                'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker" ],
                 'PortBindings' => {
                     '8888/tcp' => [
                         {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
                     ],
-                    '6006/tcp' => [
-                        {'HostPort' => "#{tensport}", 'HostIp' => 'localhost'}
-                    ],
                 },
             },
         }
@@ -3994,104 +2680,21 @@ module Cnvrg
         container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
         container.exec(command, tty: true)
-        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        # container.exec(command, tty: true)
-        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        # container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
         container.exec(command, tty: true)
-        say "#{container.id}:#{port}##{tensport}"
-      rescue => e
-        puts e
-        if e.message.include? "is not running"
-          return config_remote(image_name, port-1, tensport-1)
-        end
-
-        if container
-          container.kill()
-        end
-        return false
-      end
-    end
-
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-
-    def config_netrc(container)
-
-      login_content = options["login"]
-
-      container = Docker::Container.get(container)
-      command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-      container.exec(command, tty: true)
-      command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-      container.exec(command, tty: true)
-      command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-      container.exec(command, tty: true)
-      say "OK"
-
-    end
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
-
-
-    def config_remote_gpu(image_name, port=7654, tensport=6006)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        app_dir = options["app_dir"]
-        cmd= options["cmd"]
-
-        # image_settings = {
-        #     'Image' => "#{image_name}:latest",
-        #     'User' => 'ds',
-        #     'Cmd' => cmd,
-        #     'WorkingDir' => app_dir,
-        #     'ExposedPorts' => {
-        #         '8888/tcp' => {},
-        #     },
-        #     'HostConfig' => {
-        #         'PortBindings' => {
-        #             '8888/tcp' => [
-        #                 {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
-        #             ],
-        #             '6006/tcp' => [
-        #                 {'HostPort' => "6006", 'HostIp' => 'localhost'}
-        #             ],
-        #         },
-        #     },
-        # }
-
-        container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi  -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
-        container_id = container_id.gsub("\n", "")
-        container = Docker::Container.get(container_id)
-        # container.start()
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
-        say "#{container.id}:#{port}##{tensport}"
+        say "#{container.id}:#{port}"
       rescue => e
         if e.message.include? "is not running"
-          puts "running asgain with: #{port-1} #{tensport-1}"
-          return config_remote_gpu(image_name, port-1, tensport-1)
+          return config_remote(image_name, port-1)
         end
-
+        puts "error"
+        puts e
         if container
           container.kill()
         end
@@ -4099,8 +2702,8 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+    desc '', ''
+    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
 
     def config_flask_remote(image_name, port=80)
       local_images = Docker::Image.all
@@ -4133,54 +2736,21 @@ module Cnvrg
         container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
         container.exec(command, tty: true)
-        say "#{container.id}:#{port}"
-      rescue => e
-        pus e
-        if e.message.include? "is not running"
-          return "port is taken"
-        end
-        puts "error"
-        if container
-          container.kill()
-        end
-        return false
-      end
-    end
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-
-    def config_flask_remote_gpu(image_name, port=80)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        container_id = `nvidia-docker run -itd -p 80:80 -w /home/ds/app #{image_name}:latest /usr/local/cnvrg/start_super.sh`
-        container_id = container_id.gsub("\n", "")
-        container = Docker::Container.get(container_id)
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
         say "#{container.id}:#{port}"
       rescue => e
-        puts e
         if e.message.include? "is not running"
           return "port is taken"
         end
         puts "error"
+        puts e
         if container
           container.kill()
         end
@@ -4188,7 +2758,7 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def upload_cnvrg_image(image_name)
       verify_logged_in(false)
@@ -4198,12 +2768,12 @@ module Cnvrg
 
       path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
       @files = Cnvrg::Files.new("", "")
-      # python_arr = Cnvrg::Images.get_installed_packages("python")
-      # py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
-      # system_arr = Cnvrg::Images.get_installed_packages("system")
-      # sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      python_arr = Cnvrg::Images.get_installed_packages("python")
+      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      system_arr = Cnvrg::Images.get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
       # bash_history = Cnvrg::Images.get_bash_history
-      res = @files.upload_image(path, image_name, owner, true, true, "", "", "", "Image made by cnvrg.io team", "")
+      res = @files.upload_image(path, image_name, owner, true, true, sys, py, "", "Image made by cnvrg.io team", "")
 
     end
 
@@ -4249,7 +2819,7 @@ module Cnvrg
       end
     end
 
-    desc '', '', :hide => true
+    desc '', ''
 
     def download_image(image_name, image_slug)
       begin
@@ -4275,6 +2845,8 @@ module Cnvrg
               end
             end
 
+
+
             return dir_path
 
           else
@@ -4308,13 +2880,13 @@ module Cnvrg
         owner = Cnvrg::CLI.get_owner()
         res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
         if Cnvrg::CLI.is_response_success(res)
-          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
+          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project",  "created by", "is_public", "last updated"
           res["result"]["images"].each do |u|
             time = Time.parse(u["created_at"])
             update_at = get_local_time(time)
             created_by = u["created_by"]
 
-            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
+            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"],  created_by, u["is_public"], update_at
           end
         end
         log_end(0)
@@ -4396,7 +2968,7 @@ module Cnvrg
 
     desc 'pull_image', 'downloads and loads an image'
 
-    def pull_image(image_name)
+    def pull(image_name)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -4409,18 +2981,8 @@ module Cnvrg
         path = download_image(image_name, image["slug"])
         if path
           say "Building image", Thor::Shell::Color::BLUE
-          Docker.options[:read_timeout]=216000
-          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
-            begin
-              if (log = JSON.parse(v)) && log.has_key?("stream")
-                next if log["stream"].starts_with? "Step"
-                $stdout.puts log["stream"]
-              end
-            rescue
-            end
-
-          end
-
+          Docker.options[:read_timeout]=1200
+          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile', 't' => "#{image_name}:lastest"})
           if not image.nil?
             FileUtils.rm_rf(path)
             checks = Helpers.checkmark()
@@ -4429,13 +2991,11 @@ module Cnvrg
             return image
             log_end(0)
           else
-
             say "Could not build image", Thor::Shell::Color::RED
             log_end(1, "Could build  image")
             return false
           end
         else
-
           say "Could not download image", Thor::Shell::Color::RED
           log_end(1, "Could build  image")
           return false
@@ -4463,9 +3023,7 @@ module Cnvrg
           #
           #   end
           # end
-      rescue => e
-
-
+      rescue
         say "Couldn't build image", Thor::Shell::Color::RED
 
       rescue SignalException
@@ -4572,12 +3130,7 @@ module Cnvrg
           if url.nil? or url.empty?
             url = "https://cnvrg.io/api"
           end
-          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-          compression_path =  "#{home_dir}/.cnvrg/tmp"
-          if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
-            compression_path = config.to_h[:compression_path]
-          end
-          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
+          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url}
 
           File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
           return true
@@ -4631,7 +3184,7 @@ module Cnvrg
 
       def log_end(exit_status=0, error=nil)
         begin
-          if exit_status != 0
+          if exit_status==1
             $LOG.error exit_status: exit_status, error: error
           else
             $LOG.info exit_status: exit_status
@@ -4643,6 +3196,7 @@ module Cnvrg
       def self.is_response_success(response, should_exit=true)
         if response.nil?
           if !Cnvrg::Helpers.internet_connection?
+            # Cnvrg::CLI.log_end(1,"no internet connection")
             say("<%= color('Error:You seems to be offline', RED) %>")
           else
             say("<%= color('Error', RED) %>")
@@ -4677,26 +3231,12 @@ module Cnvrg
           return owner
         end
       end
-
       def get_base_url
         home_dir = File.expand_path('~')
 
         config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
         api = config.to_h[:api]
-        return api.gsub!("/api", "")
-      end
-      def  get_compression_path
-        home_dir = File.expand_path('~')
-
-        config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
-        compression_path = config.to_h[:compression_path]
-        if compression_path.nil?
-          compression_path = "#{home_dir}/.cnvrg/tmp/"
-        end
-        if !compression_path.ends_with? "/"
-          compression_path = compression_path + "/"
-        end
-        return compression_path
+        return api.gsub!("/api","")
       end
 
       def get_project_home
@@ -4750,6 +3290,7 @@ module Cnvrg
           end
           $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
         rescue => e
+          # puts e
         end
       end
 
@@ -4777,9 +3318,6 @@ module Cnvrg
 
         config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
         version_date = config.to_h[:version_last_check]
-        if version_date.nil?
-          version_date = get_start_day()
-        end
         next_day = get_start_day+ 86399
         if not (version_date..next_day).cover?(Time.now)
           if should_update_version()
@@ -4805,15 +3343,11 @@ module Cnvrg
       def is_cnvrg_dir(dir=Dir.pwd)
         current_dir = dir
         home_dir = File.expand_path('~')
-        if current_dir.eql? home_dir
-          return false
-        end
         is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
         until is_cnvrg == true
           current_dir = File.expand_path("..", current_dir)
           is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
-          if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
-            is_cnvrg = false
+          if File.expand_path("..", current_dir).eql? home_dir
             break
           end
         end
@@ -4824,19 +3358,6 @@ module Cnvrg
         end
       end
 
-      def data_dir_include()
-        all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
-        all_dirs.flatten!
-        all_dirs.each do |a|
-          if a.include? "/.cnvrg"
-            ignore = File.dirname(a)
-            return ignore
-          end
-        end
-        return nil
-
-      end
-
       def verify_software_installed(software)
         begin
           install_url = Cnvrg::CLI::INSTALLATION_URLS[software.to_sym]
@@ -5012,151 +3533,8 @@ module Cnvrg
 
       end
 
-      def update_deleted(deleted)
-        final = []
-        deleted.each do |d|
-          all_subs = d.split("/")
-          to_add = true
-          value = all_subs[0]
-          all_subs.each_with_index do |a, i|
-            if final.include? value+"/"
-              to_add = false
-              break
-            end
-            value = value+"/"+all_subs[i+1] if i <all_subs.size-1
-          end
-          final << d if to_add
-
-
-        end
-
-
-        return final
-      end
-
-      def get_cmd_path_in_dir(main_dir, sub_dir)
-        first = Pathname.new main_dir
-        second = Pathname.new sub_dir
-        relative = second.relative_path_from first
-        if relative.eql? "."
-          return ""
-        else
-          return relative
-        end
-      end
-
-      def create_tar(path_in, path_out, tar_files,no_compression=false)
-        #The cd is meant for cases when running cnvrg data uplaod not in the main folder
-        begin
-          if no_compression
-            `cd #{path_in} && tar -cf #{path_out} -T #{tar_files}`
-          else
-            `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
-          end
-        rescue => e
-          puts "Exception while compressing data: #{e.message}"
-        end
-
-        return $?.success?
-      end
-
-      def extarct_tar(file_path, dir_path)
-        `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
-        return $?.success?
-      end
-      def cpu_usage
-        cpu_usage = 0.0
-        begin
-          cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
-        if !cpu.nil?
-          cpu_usage = cpu.to_f
-        end
-        rescue
-          cpu_usage = 0.0
-        end
-
-        return cpu_usage
-      end
-      def memory_usage
-        prec = 0.0
-        begin
-        used = `free -mt |grep Mem: |awk '{print $3}'`
-        total = `free -mt |grep Mem: |awk '{print $2}'`
-
-        used_f = used.to_f if !used.nil?
-        total_f = total.to_f if !total.nil?
-          prec = (used_f / total_f)*100
-          prec = prec.round(2)
-        rescue
-          prec = 0.0
-        end
-        return prec
-
-
-      end
-      def gpu_util
-        gpu = [0.0, 0.0]
-        begin
-        gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
-
-        if !gpu_stats.nil?
-          gpu = gpu_stats.strip
-          gpu = gpu_stats.gsub!("%","").split(",")
-          gpu[0] = gpu[0].to_f
-          gpu[1] = gpu[1].to_f
-          return gpu
-        end
-
-        rescue
-          return gpu
-        end
-
-
-      end
-      def usage_metrics_in_docker(docker_id)
-        res = [0.0,0.0]
-        begin
-          if docker_id.nil?
-            docker_id = `cat /etc/hostname`
-          end
-          stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
-          if !stats.nil?
-            conv = stats.split(",")
-            cpu = conv[0].gsub!("%","").to_f
-            memory = conv[1].gsub!("%","").to_f
-            res = [cpu,memory]
-            return res
-          end
-        rescue
-          return res
-        end
-
-
-
-      end
-
-      def with_progress_bar
-        not_done = true
-        upload_progress_bar = ProgressBar.create(:title => "Upload progress",
-                                                 :format => '%a <%B> %p%% %t',
-                                                 :starting_at => 0,
-                                                 :total => 600)
-        pb = Thread.new do
-          while not_done do
-            upload_progress_bar.increment
-            sleep(1)
-
-          end
-        end
 
-        yield.tap do # After yielding to the block, save the return value
-          not_done = false # Tell the thread to exit, cleaning up after itself…
-          pb.join # …and wait for it to do so.
-        end
-      end
     end
-
-
   end
 end