cnvrg 0.0.14.0 → 0.0.15

data/lib/cnvrg/cli.rb

@@ -7,7 +7,7 @@ require 'uri'
 require 'open-uri'
 require 'json'
 require 'yaml'
-require 'digest' # sha1
+require 'digest' # sha1up
 require "highline/import"
 require 'socket'
 include Open4
@@ -18,6 +18,10 @@ require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
 require 'cnvrg/Images'
+require 'cnvrg/dataset'
+require 'cnvrg/datafiles'
+require 'cnvrg/data'
+require 'cnvrg/ssh'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -29,28 +33,116 @@ require 'fileutils'
 require 'zip'
 require 'active_support/all'
 require 'thor'
-require 'pry'
+require 'pathname'
+require 'enumerator'
+
+class Thor
+  module Base
+    def initialize(args = [], local_options = {}, config = {})
+      parse_options = Thor.class_options
+
+      # The start method splits inbound arguments at the first argument
+      # that looks like an option (starts with - or --). It then calls
+      # new, passing in the two halves of the arguments Array as the
+      # first two parameters.
+
+      command_options = config.delete(:command_options) # hook for start
+      parse_options = parse_options.merge(command_options) if command_options
+      if local_options.is_a?(Array)
+        array_options = local_options
+        hash_options = {}
+      else
+        # Handle the case where the class was explicitly instantiated
+        # with pre-parsed options.
+        array_options = []
+        hash_options = local_options
+
+      end
+
+      # Let Thor::Options parse the options first, so it can remove
+      # declared options from the array. This will leave us with
+      # a list of arguments that weren't declared.
+      stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
+      opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
+      real_options = []
+
+      real_args = [].replace(array_options)
+      if local_options.is_a? (Array) and !local_options.empty? and args.empty?
+        array_options.each_with_index do |p, i|
+          opt = p
+          if p.include? "="
+            opt = p.split("=")[0]
+          end
+          option = is_option(parse_options.values, opt)
+          if !option
+            break
+          else
+            real_options << p
+            real_args.delete(p)
+            if !p.include? "=" and option.type != :boolean
+              if i+1< array_options.size
+                real_options << array_options[i+1]
+                real_args.delete(array_options[i+1])
+              end
+            end
+
+          end
+
+        end
+
+        args = real_args
+      else
+        if !args.empty? and local_options.is_a? Array and !local_options.empty?
+          args = args + local_options
+        else
+          args = args.flatten()
+        end
+
+      end
+
+
+      self.options = opts.parse(real_options)
+      self.options = config[:class_options].merge(options) if config[:class_options]
+
+      # If unknown options are disallowed, make sure that none of the
+      # remaining arguments looks like an option.
+      opts.check_unknown! if Thor.check_unknown_options?(config)
+
+      # Add the remaining arguments from the options parser to the
+      # arguments passed in to initialize. Then remove any positional
+      # arguments declared using #argument (this is primarily used
+      # by Thor::Group). Tis will leave us with the remaining
+      # positional arguments.
+      to_parse = args
+      to_parse += opts.remaining unless self.class.strict_args_position?(config)
+      thor_args = Thor::Arguments.new(self.class.arguments)
+      thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
+      @args = thor_args.remaining
+    end
+
+    def is_option (options, p)
+      options.each do |o|
+        if !o.aliases.nil?
+          if (o.aliases.is_a? Array and o.aliases.include? p) or (o.aliases.is_a? Array and o.aliases.size ==1 and o.aliases[0].split(",").include? p) or o.switch_name.eql? p
+            return o
+          end
+        end
+
+      end
+      return false
+    end
+
+  end
+end
+
 
-# DEV VERSION
-#
 module Cnvrg
+
   class CLI < Thor
 
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
     IP="localhost"
     PORT=7654
-    desc '', ''
-
-    def printable_commands(all = true, subcommand = false)
-      (all ? all_commands : commands).map do |_, command|
-        next if command.hidden? or (command.description.empty? and command.usage.empty?)
-        item = []
-        item << banner(command, false, subcommand)
-        item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
-        item
-      end.compact
-    end
-
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
       def is_thor_reserved_word?(word, type)
@@ -58,9 +150,10 @@ module Cnvrg
         super
       end
     end
+    desc "data", "upload and manage datasets", :hide => true
+    subcommand "data", Data
 
-    desc "", ""
-    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
+    desc "", "", :hide => true
 
     def test
       # image_settings = {
@@ -78,11 +171,10 @@ module Cnvrg
       #         },
       #     },
       # }
-      container = Docker::Container.get('b4d64bf83f41')
-      s = "/leah/1/2/3/4/5"
-      command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-      puts container.exec(command, tty: true)
-
+      # container = Docker::Container.get('b4d64bf83f41')
+      # s = "/leah/1/2/3/4/5"
+      # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+      # puts container.exec(command, tty: true)
     end
 
 
@@ -95,7 +187,7 @@ module Cnvrg
 
     map %w(-v --version) => :version
 
-    desc 'set api url', 'set api url'
+    desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
 
     def set_api_url(url)
       home_dir = File.expand_path('~')
@@ -111,15 +203,19 @@ module Cnvrg
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
         owner = config.to_h[:owner]
+        compression_path =  "#{File.expand_path('~')}/.cnvrg/tmp"
 
         say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
         if config.empty?
-          config = {owner: "", username: "", version_last_check: get_start_day(), api: url}
+          config = {owner: "", username: "", version_last_check: get_start_day(), api: url,compression_path: compression_path }
         else
-          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url}
+          if !config.to_h[:compression_path].nil?
+            compression_path = config.to_h[:compression_path]
+          end
+          config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
         end
         res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
-        if Cnvrg::CLI.is_response_success(res)
+        if Cnvrg::CLI.is_response_success(res,false)
 
           checks = Helpers.checkmark
 
@@ -137,7 +233,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def set_remote_api_url(owner, current_user, url)
       home_dir = File.expand_path('~')
@@ -152,7 +248,9 @@ module Cnvrg
           FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url}
+
+        compression_path =  "#{home_dir}/.cnvrg/tmp"
+        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url,compression_path:compression_path}
         File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
         say "Done"
       rescue
@@ -162,57 +260,108 @@ module Cnvrg
 
     map %w(-api --api) => :set_api_url
 
-    desc 'set default owner', 'set default owner'
+    desc 'set_default_owner', 'set default owner'
 
     def set_default_owner
+      begin
+        path = File.expand_path('~')+"/.cnvrg/config.yml"
+        if !File.exist?(path)
+          say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
 
-      path = File.expand_path('~')+"/.cnvrg/config.yml"
-      if !File.exist?(path)
-        say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
+          exit(0)
+        end
+        config = YAML.load_file(path)
 
-        exit(0)
-      end
-      config = YAML.load_file(path)
+        username = config.to_h[:username]
+        res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
+        if Cnvrg::CLI.is_response_success(res)
+          owner = username
+          result = res["result"]
+          owners = result["owners"]
+          urls = result["urls"]
+          choose_owner = result["username"]
+          if owners.empty?
+          else
+            owners << choose_owner
+            chosen = false
+            while !chosen
+              owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
+              choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
+
+              if choose_owner =~ /[[:digit:]]/
+                ow_index = choose_owner.to_i-1
+                if ow_index<0 or ow_index >= owners.size
+                  say "No such owner, please choose again", Thor::Shell::Color::BLUE
+                  chosen = false
+                  next
+                end
+                choose_owner = owners[choose_owner.to_i-1]
+                chosen = true
 
-      username = config.to_h[:username]
-      res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
-      if Cnvrg::CLI.is_response_success(res)
-        owner = username
-        result = res["result"]
-        owners = result["owners"]
-        urls = result["urls"]
-        choose_owner = result["username"]
+              else
+
+                owners_lower = owners.map { |o| o.downcase }
+                ow_index = owners_lower.index(choose_owner.downcase)
+                if ow_index.nil?
+                  say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+                else
+                  chosen = true
+                end
+              end
 
-        if owners.empty?
-        else
-          owners << choose_owner
-          chosen = false
-          while !chosen
-            choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
-
-            owners_lower = owners.map { |o| o.downcase }
-            ow_index = owners_lower.index(choose_owner.downcase)
-            if ow_index.nil?
-              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-            else
-              chosen = true
             end
+
+
+          end
+          if set_owner(choose_owner, result["username"], urls[ow_index])
+            say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
+          else
+            say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
           end
+        end
+      rescue SignalException
+        say "\nAborting"
+        exit(1)
+      end
+    end
 
+    desc 'set_compression_path', 'set compression path'
+    method_option :reset, :type => :boolean, :aliases => ["-r","--reset"], :default => false
 
+    def set_compression_path(*compression_path)
+      begin
+        if (compression_path.nil? or compression_path.empty?) and options["reset"]
+          compression_path =  ["#{File.expand_path('~')}/.cnvrg/tmp"]
         end
-        if set_owner(owner, username,urls[ow_index])
-          say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
-        else
-          say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
+        compression_path = compression_path.join(" ")
+        if !Dir.exist? compression_path
+          say "Couldn't find #{compression_path}, please make sure it exist", Thor::Shell::Color::RED
+          exit(0)
+        end
+
+        home_dir = File.expand_path('~')
+        path = "#{home_dir}/.cnvrg/config.yml"
+        if !File.exist?(path)
+          say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
+
+          exit(0)
         end
+        config = YAML.load_file(path)
+        config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
+                  version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
+        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config_new.to_yaml }
+        checks = Helpers.checkmark
+        say "#{checks} Done", Thor::Shell::Color::GREEN
+
+      rescue SignalException
+        say "\nAborting"
+        exit(1)
       end
     end
 
-    map %w(--set-default-owner) => :set_default_owner
 
 
-    desc 'login', 'Authenticate with cnvrg.io and store credentials'
+    desc 'login', 'Authenticate with cnvrg.io platform'
 
     def login
       begin
@@ -240,25 +389,46 @@ module Cnvrg
           owners = result["owners"]
           urls = result["urls"]
           choose_owner = result["username"]
-
+          ow_index = 0
           if owners.empty?
+            choose_owner = result["username"]
           else
-            owners << choose_owner
-            chosen = false
-            while !chosen
-              choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
+            choose_owner = owners[0]
+          end
 
-              owners_lower = owners.map { |o| o.downcase }
-              ow_index = owners_lower.index(choose_owner.downcase)
-              if ow_index.nil?
-                say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-              else
-                chosen = true
-              end
-            end
+          # if owners.empty?
+          # else
+          #   owners << choose_owner
+          #   chosen = false
+          #   while !chosen
+          #     owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
+          #     choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
+          #
+          #     if choose_owner =~ /[[:digit:]]/
+          #       ow_index = choose_owner.to_i-1
+          #       if ow_index<0 or ow_index >= owners.size
+          #         say "No such owner, please choose again", Thor::Shell::Color::BLUE
+          #         chosen = false
+          #         next
+          #       end
+          #       choose_owner = owners[choose_owner.to_i-1]
+          #       chosen = true
+          #
+          #     else
+          #
+          #       owners_lower = owners.map { |o| o.downcase }
+          #       ow_index = owners_lower.index(choose_owner.downcase)
+          #       if ow_index.nil?
+          #         say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+          #       else
+          #         chosen = true
+          #       end
+          #     end
+          #
+          #   end
 
 
-          end
+          # end
 
           if set_owner(choose_owner, result["username"], urls[ow_index])
             say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
@@ -274,6 +444,11 @@ module Cnvrg
 
           exit(1)
         end
+      rescue
+
+        say "Error Occurred, aborting", Thor::Shell::Color::RED
+        logout()
+        exit(1)
       rescue SignalException
 
         say "\nAborting"
@@ -303,7 +478,6 @@ module Cnvrg
 
     def me()
       begin
-
         verify_logged_in(false)
         log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
@@ -314,6 +488,7 @@ module Cnvrg
         end
 
         log_end(0)
+
       rescue SignalException
         log_end(-1)
 
@@ -324,8 +499,8 @@ module Cnvrg
 
     ## Projects
     desc 'new', 'Create a new cnvrg project'
-    method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
-    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
+    method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
+    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
 
     def new(project_name)
       begin
@@ -393,8 +568,8 @@ module Cnvrg
         say "created project successfully", Thor::Shell::Color::GREEN
         say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
       rescue => e
-        puts e
-        log_end(-1)
+        log_end(-1, e.message)
+        say "Error occurred, aborting", Thor::Shell::Color::RED
         if Dir.exist? working_dir
 
           @project.revert(working_dir)
@@ -412,6 +587,7 @@ module Cnvrg
         exit(1)
       end
     end
+
     desc 'set_image', 'set_image for a project'
 
     def set_image(docker_image)
@@ -448,9 +624,10 @@ module Cnvrg
       @image.update_image_activity(nil, nil)
 
     end
-    desc 'link', 'Link current directory to a cnvrg project'
-    method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
-    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
+
+    desc 'link', 'Link current directory to a new cnvrg project'
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
 
     def link
       begin
@@ -474,7 +651,7 @@ module Cnvrg
         end
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Project.link(owner,project_name, docker)
+        if Project.link(owner, project_name, docker)
           path = Dir.pwd
           @project = Project.new(path)
           @project.generate_idx()
@@ -531,143 +708,834 @@ module Cnvrg
         exit(1)
       end
     end
-    desc 'unlink','Unlink a porject'
-    def unlink
-      verify_logged_in(false)
-      log_start(__method__, args, options)
-      working_dir = is_cnvrg_dir()
-      list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
-      FileUtils.rm_rf list_to_del
+
+    desc 'data init', 'Init dataset directory'
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+
+    def init_data(public)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        dataset_name =File.basename(Dir.getwd)
+        if File.directory?(Dir.getwd+"/.cnvrg")
+          config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
+          say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
+
+          exit(0)
+        end
+        say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
+
+        working_dir = Dir.getwd
+        owner = CLI.get_owner
+        if Dataset.init(owner, dataset_name, options["public"])
+          path = Dir.pwd
+          @dataset = Dataset.new(path)
+
+          url = @dataset.url
+          check = Helpers.checkmark
+          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
+          say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
+          log_end(0)
+
+        else
+          log_end(1, "can't create dataset")
+          @dataset.revert(working_dir) unless @dataset.nil?
+          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          exit(0)
+        end
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
     end
 
-    desc 'clone', 'Clone a project'
-    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
-    method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
+    desc 'data clone', 'Clone dataset'
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
 
-    def clone(project_url)
+    def clone_data(dataset_url)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-        url_parts = project_url.split("/")
-        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
+        url_parts = dataset_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index+1]
         owner = url_parts[project_index-1]
-        remote = options["remote"] || false
-        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
-        Cnvrg::CLI.is_response_success(response)
-        response = JSON.parse response["result"]
-        project_name = response["title"]
-        commit_to_clone = options["commit"] || nil
-
-        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
-        clone_resp = false
-        if remote
-          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
-          project_home = Dir.pwd
-        else
-          if (Dir.exists? project_name)
-            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
-            if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
-              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
-              log_end(1, "conflict with dir #{project_name}")
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
 
-              exit(1)
-            end
+        Cnvrg::CLI.is_response_success(response)
+        dataset_name = response["result"]["name"]
+        if (Dir.exists? dataset_name)
+          say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
+          if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+            say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
+            log_end(1, "conflict with dir #{dataset_name}")
 
+            exit(1)
           end
-          clone_resp = Project.clone_dir(slug, owner, project_name)
-          project_home = Dir.pwd+"/"+project_name
-
 
         end
+        if Dataset.clone(owner, dataset_name, slug)
+          say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
 
-        if clone_resp
-          @project = Project.new(project_home)
-          @files = Cnvrg::Files.new(@project.owner, slug)
-          response = @project.clone(remote, commit_to_clone)
-          Cnvrg::CLI.is_response_success response
-          working_dir = project_home
-          docker_image = response["result"]["image"]
-          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-          File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
-          if !docker_image.nil? and !docker_image.empty? and !remote
-            local_images = Docker::Image.all
-            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
-            if docker_image_local.size == 0
-              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-                image = pull(docker_image)
-                if image
-                  say "downloaded image: #{docker_image}"
-                  @image = Images.new(working_dir, docker_image)
-                else
-                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
-                  @project.revert(working_dir)
-                  exit(1)
-                end
-              end
+          commit_to_clone = options["commit"] || nil
+          working_dir = "#{Dir.pwd}/#{dataset_name}"
+          @dataset = Dataset.new(working_dir)
+          @dataset.generate_idx()
+          say "Downloading data", Thor::Shell::Color::BLUE
 
-            elsif docker_image_local.size == 1
-              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, docker_image_local[0])
-            elsif docker_image_local.size >1
-              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
-              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-              @image = Images.new(working_dir, image_name)
-            end
 
-          end
+          download_data(false, false, path = working_dir, in_dir=false)
 
-          successful_changes = []
-          say "Downloading files", Thor::Shell::Color::BLUE
-          if !response["result"]["tree"].nil?
-            response["result"]["tree"].each do |f|
-              relative_path = f[0].gsub(/^#{@project.local_path}/, "")
-              if f[0].end_with? "/"
-                # dir
-                if @files.download_dir(f[0], relative_path, project_home)
-                  successful_changes << relative_path
-                end
-              else
-                # blob
-                if @files.download_file_s3(f[0], relative_path, project_home)
-                  successful_changes << relative_path
-                end
-              end
-            end
-          end
 
-          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
+          check = Helpers.checkmark
+          say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
           log_end(0)
-        else
-          log_end(1, "can't create directory")
 
-          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
-          exit(1)
+        else
+          log_end(1, "can't create dataset")
+          @dataset.revert(working_dir) unless @dataset.nil?
+          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          exit(0)
         end
       rescue SignalException
         log_end(-1)
+
         say "\nAborting"
         exit(1)
       end
-
     end
 
+    desc 'init_data_container', 'Init dataset directory', :hide => true
+    method_option :login_content, :type => :string, :aliases => ["-l"], :default => ""
 
-    desc 'status', 'Show the working tree status'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-
-
-    def status
+    def init_data_container(container)
       begin
-        verify_logged_in()
-        log_start(__method__, args, options)
-        @project = Project.new(get_project_home)
-        new_branch = options["new_branch"] || false
+        login_content = options["login_content"]
 
-        result = @project.compare_idx(new_branch)["result"]
+        container = Docker::Container.get(container)
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        container.exec(command, tty: true)
+
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'data_snap', 'Init dataset directory', :hide => true
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+
+    def snap_data
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+
+        owner = CLI.get_owner
+        path = Dir.pwd
+        @dataset = Dataset.new(path)
+
+        log_end(0)
+
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'data_snap', 'Init dataset directory', :hide => true
+
+    def data_init_container(owner, dataset_slug, dataset_name)
+
+      if Dataset.init_container(owner, dataset_slug, dataset_name)
+
+        say "init finished successfully", Thor::Shell::Color::GREEN
+        log_end(0)
+
+      else
+        log_end(1, "can't create dataset")
+        say "error creating dataset, please contact support.", Thor::Shell::Color::RED
+        exit(0)
+      end
+    end
+
+    desc 'data download', 'pull data'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def download_data(verbose, sync, path=Dir.pwd, in_dir=true)
+      begin
+        verify_logged_in(in_dir)
+        log_start(__method__, args, options)
+        if path.nil? or path.empty?
+          path = Dir.pwd
+        end
+        dataset_dir = is_cnvrg_dir(path)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        new_branch = options["new_branch"] || false
+
+        res = @dataset.compare_idx(new_branch)["result"]
+
+        result = res["tree"]
+
+        commit = res["commit"]
+        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
+          log_end(0)
+          return true
+        end
+        result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
+
+        delete = result["result"]["delete"]
+        commits = result["result"]["commits"]
+        updated_idx = result["result"]["idx"]
+        commits.each do |c|
+          file_name = @files.download_data_file(c, dataset_dir)
+
+          if file_name.eql? false or file_name.nil?
+            ##RETRY
+            next
+          end
+          file_path = "#{dataset_dir}/#{file_name}"
+          success = extarct_tar(file_path, dataset_dir)
+          if !success
+            #TODO: ?
+          end
+
+          FileUtils.rm_rf([file_path])
+
+        end
+        to_delete = []
+        delete.each do |d|
+          to_delete << "#{dataset_dir}/#{d}"
+        end
+        FileUtils.rm_rf(to_delete)
+
+        @dataset.update_idx(updated_idx)
+
+
+        # result["conflicts"].each do |f|
+        #   relative_path = f.gsub(/^#{@dataset.local_path}/, "")
+        #   if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
+        #     successful_changes << relative_path
+        #   end
+        #
+        # end
+        # update idx with latest commit
+        # @dataset.update_idx_with_commit!(commit)
+        check = Helpers.checkmark()
+        say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
+
+
+        log_end(0)
+      end
+    rescue
+      log_end(-1)
+
+      say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+      exit(1)
+    rescue SignalException
+      log_end(-1)
+      say "\nAborting", Thor::Shell::Color::BLUE
+      exit(1)
+    end
+
+
+    desc 'upload_data', 'Upload data files', :hide => true
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def upload_data(sync=false, direct=false)
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        ignore = options[:ignore] || []
+        if !@dataset.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
+        result = @dataset.compare_idx(false)
+
+        commit = result["result"]["commit"]
+        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+          log_end(0)
+
+          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+          exit(1)
+        end
+
+        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
+        result = result["result"]["tree"]
+        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
+        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
+        #   if all.size == 1
+        #     num = "conflict"
+        #   else
+        #     num =  "conflicts"
+        #   end
+        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
+        #   say "#{all.join("\n")}"
+        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
+        #   exit(1)
+        #
+        # end
+        check = Helpers.checkmark()
+
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_end(0)
+          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+        if options["verbose"]
+          if update_total == 1
+            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
+          else
+            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
+          end
+        else
+          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
+
+        end
+
+        # Start commit
+
+        commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
+        # upload / update
+        begin
+          (result["added"] + result["updated_on_local"]).each do |f|
+            absolute_path = "#{@dataset.local_path}/#{f}"
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            if File.directory?(absolute_path)
+              resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+              if resDir
+                update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+
+              if res
+                update_count += 1
+                successful_updates<< relative_path
+              else
+                @files.rollback_commit(commit_sha1)
+                log_end(1, "can't upload, Rolling Back all changes")
+                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+                exit(0)
+              end
+            end
+          end
+
+          # delete
+          deleted = update_deleted(result["deleted"])
+          deleted.each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            if relative_path.end_with?("/")
+              if @files.delete_dir(f, relative_path, commit_sha1)
+                # update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              if @files.delete_file(f, relative_path, commit_sha1)
+                # update_count += 1
+                successful_updates<< relative_path
+              end
+            end
+          end
+
+        rescue SignalException
+          log_end(-1)
+          @files.rollback_commit(commit_sha1)
+          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          log_end(-1, e.message)
+          @files.rollback_commit(commit_sha1)
+          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          exit(0)
+        end
+        if !result["deleted"].nil? and !result["deleted"].empty?
+          update_count += result["deleted"].size
+        end
+        if update_count == update_total
+          res = @files.end_commit(commit_sha1)
+          if (Cnvrg::CLI.is_response_success(res, false))
+            # save idx
+            begin
+              @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
+
+              @dataset.update_idx_with_commit!(commit_sha1)
+            rescue => e
+              log_end(-1, e.message)
+              @files.rollback_commit(commit_sha1)
+              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
+              exit(1)
+
+            end
+            if options["verbose"]
+              say "#{check} Done", Thor::Shell::Color::BLUE
+              if successful_updates.size >0
+                say "Updated:", Thor::Shell::Color::GREEN
+                suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+                say suc.join("\n"), Thor::Shell::Color::GREEN
+              end
+              if successful_deletions.size >0
+                say "Deleted:", Thor::Shell::Color::GREEN
+                del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+                say del.join("\n"), Thor::Shell::Color::GREEN
+              end
+              say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+            else
+              if (options["sync"] or sync) and direct
+                say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
+
+              else
+                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
+              end
+
+            end
+
+            log_end(0)
+          else
+            @files.rollback_commit(commit_sha1)
+            log_end(1, "error. Rolling Back all changes")
+            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
+          end
+        else
+          log_end(1, "error. Rolling Back all changes")
+          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
+
+          @files.rollback_commit(commit_sha1)
+        end
+      rescue => e
+
+        log_end(-1)
+
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      end
+
+    end
+
+    desc 'data upload', 'push data'
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
+
+    def upload_data_tar(ignore, verbose, sync,no_compression)
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        if !@dataset.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED
+          exit(1)
+        end
+        say "Checking dataset", Thor::Shell::Color::BLUE
+        local_idx = @dataset.generate_idx
+        result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
+
+
+        commit = result["result"]["commit"]
+        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+          log_end(0)
+
+          say "Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::YELLOW
+          exit(1)
+        end
+
+        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
+        result = result["result"]["tree"]
+        check = Helpers.checkmark()
+
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_end(0)
+          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+
+        # Start commit
+        res = @files.start_commit(false)["result"]
+        commit_sha1 =res["commit_sha1"]
+        commit_time = res["commit_time"]
+        # upload / update
+        begin
+          (result["added"] + result["updated_on_local"]).each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            successful_updates<< relative_path
+            update_count += 1
+          end
+
+          # delete
+          deleted = update_deleted(result["deleted"])
+          deleted.each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            successful_updates<< relative_path
+          end
+          @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
+
+          @dataset.update_idx_with_commit!(commit_sha1)
+          say "Compressing data", Thor::Shell::Color::BLUE
+
+          home_dir = File.expand_path('~')
+          compression_path = get_compression_path
+          tar_path = "#{compression_path}#{@dataset.slug}_#{commit_sha1}.tar.gz"
+          tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
+          tar_files = (result["added"] + result["updated_on_local"]).join("\n")
+          File.open(tar_files_path, 'w') { |f| f.write tar_files }
+          is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
+          if !is_tar
+            say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
+            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+            @files.rollback_commit(commit_sha1)
+            say "Rolling Back all changes.", Thor::Shell::Color::RED
+            exit(1)
+          end
+          say "Uploading data", Thor::Shell::Color::BLUE
+          res = false
+            res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
+
+          if res
+            say "Commiting data", Thor::Shell::Color::BLUE
+
+            cur_idx = @dataset.get_idx.to_h
+
+            res = @files.end_commit_tar(commit_sha1, cur_idx)
+            if !Cnvrg::CLI.is_response_success(res, false)
+              FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+              FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+              @files.rollback_commit(commit_sha1)
+              say "Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED
+              exit(1)
+            end
+
+          else
+            FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+            FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+            @files.rollback_commit(commit_sha1)
+            say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+            exit(1)
+          end
+
+
+          # delete
+          FileUtils.rm_rf([tar_path, tar_files_path])
+
+        rescue SignalException
+          log_end(-1)
+          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+          @files.rollback_commit(commit_sha1)
+          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          puts e.message
+          if !Cnvrg::Helpers.internet_connection?
+            say "Seems there is no internet connection", Thor::Shell::Color::RED
+
+          end
+          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+          log_end(-1, e.message)
+          @files.rollback_commit(commit_sha1)
+          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          exit(0)
+        end
+        if verbose
+          say "#{check} Done", Thor::Shell::Color::BLUE
+          if successful_updates.size >0
+            say "Updated:", Thor::Shell::Color::GREEN
+            suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+            say suc.join("\n"), Thor::Shell::Color::GREEN
+          end
+          if successful_deletions.size >0
+            say "Deleted:", Thor::Shell::Color::GREEN
+            del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+            say del.join("\n"), Thor::Shell::Color::GREEN
+          end
+
+
+          say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+        else
+          say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
+        end
+
+
+      rescue => e
+        log_end(-1, e.message)
+
+
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      end
+
+      log_end(0)
+
+    end
+
+
+    desc 'unlink', 'Unlink a project from current directory', :hide => true
+
+    def create_volume
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      @dataset.create_volume()
+
+    end
+
+    desc 'data list', 'List all dataset you currently have'
+
+    def list_dataset
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      owner = @dataset.owner
+      if owner.nil? or owner.empty?
+        owner = CLI.get_owner()
+      end
+      result = @dataset.list(owner)
+      list = result["result"]["list"]
+
+      print_table(list)
+
+    end
+
+    desc 'data commits', 'List all commits for a specific dataset'
+
+    def list_dataset_commits()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      result = @dataset.list_commits()
+      list = result["result"]["list"]
+
+      print_table(list)
+
+    end
+
+    desc 'commits', 'List all commits for a specific dataset'
+
+    def list_commits()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+
+      project_dir = is_cnvrg_dir(Dir.pwd)
+      @project = Project.new(project_dir)
+      result = @project.list_commits()
+      list = result["result"]["list"]
+
+      print_table(list)
+
+    end
+
+
+    desc 'unlink', 'Unlink a project from current directory'
+
+    def unlink
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir()
+      list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
+      FileUtils.rm_rf list_to_del
+    end
+
+    desc 'clone', 'Clone project'
+    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
+
+    def clone(project_url)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        url_parts = project_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
+        slug = url_parts[project_index+1]
+        owner = url_parts[project_index-1]
+        remote = options["remote"] || false
+        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
+        Cnvrg::CLI.is_response_success(response)
+        response = JSON.parse response["result"]
+        project_name = response["title"]
+        commit_to_clone = options["commit"] || nil
+
+        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
+        clone_resp = false
+        if remote
+          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
+          project_home = Dir.pwd
+        else
+          if (Dir.exists? project_name)
+            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
+            if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
+              log_end(1, "conflict with dir #{project_name}")
+
+              exit(1)
+            end
+
+          end
+          clone_resp = Project.clone_dir(slug, owner, project_name)
+          project_home = Dir.pwd+"/"+project_name
+
+
+        end
+
+        if clone_resp
+          @project = Project.new(project_home)
+          @files = Cnvrg::Files.new(@project.owner, slug)
+          response = @project.clone(remote, commit_to_clone)
+          Cnvrg::CLI.is_response_success response
+          working_dir = project_home
+          docker_image = response["result"]["image"]
+          current_commit = response["result"]["commit"]
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          if !docker_image.nil? and !docker_image.empty? and !remote
+            local_images = Docker::Image.all
+            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
+            if docker_image_local.size == 0
+              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+                image = pull(docker_image)
+                if image
+                  say "downloaded image: #{docker_image}"
+                  @image = Images.new(working_dir, docker_image)
+                else
+                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  @project.revert(working_dir)
+                  exit(1)
+                end
+              end
+
+            elsif docker_image_local.size == 1
+              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, docker_image_local[0])
+            elsif docker_image_local.size >1
+              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, image_name)
+            end
+
+          end
+
+          successful_changes = []
+          say "Downloading files", Thor::Shell::Color::BLUE
+          if !response["result"]["tree"].nil?
+            response["result"]["tree"].each do |f|
+              relative_path = f[0].gsub(/^#{@project.local_path}/, "")
+              if f[0].end_with? "/"
+                # dir
+                if @files.download_dir(f[0], relative_path, project_home)
+                  successful_changes << relative_path
+                end
+              else
+                # blob
+
+                if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
+                  successful_changes << relative_path
+                end
+              end
+            end
+          end
+
+          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
+          log_end(0)
+        else
+          log_end(1, "can't create directory")
+
+          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
+          exit(1)
+        end
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      end
+
+    end
+
+
+    desc 'status', 'Show the working tree status'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+
+
+    def status
+      begin
+        verify_logged_in()
+        log_start(__method__, args, options)
+        @project = Project.new(get_project_home)
+        new_branch = options["new_branch"] || false
+
+        result = @project.compare_idx(new_branch)["result"]
         commit = result["commit"]
         result = result["tree"]
         say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
+
         if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
           say "Project is up to date", Thor::Shell::Color::GREEN
           log_end(0)
@@ -714,14 +1582,36 @@ module Cnvrg
       end
     end
 
+    desc '', '', :hide => true
+
+    def revert_exp
+      begin
+        log_start(__method__, args, options)
+        @project = Project.new(get_project_home)
+
+        result = @project.compare_idx(false)["result"]
+        result = result["tree"]
+        if result["added"].size > 0
+          FileUtils.rm_rf(result["added"])
+        end
+        say "Changes were removed successfully", Thor::Shell::Color::GREEN
+
+
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
 
     desc 'upload', 'Upload updated files'
-    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def upload(link=false, sync=false)
+    def upload(link=false, sync=false, direct=false, ignore_list="")
 
       begin
         verify_logged_in(true)
@@ -730,12 +1620,30 @@ module Cnvrg
         @project = Project.new(get_project_home)
 
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        ignore = options[:ignore] || []
+        ignore = options[:ignore] || ""
+
+        if ignore.nil? or ignore.empty?
+          ignore = ignore_list
+        end
+        data_ignore = data_dir_include()
+        if !data_ignore.nil?
+          if ignore.nil? or ignore.empty?
+            ignore = data_ignore
+          else
+            ignore ="#{ignore},#{data_ignore}"
+          end
+        end
         if !@project.update_ignore_list(ignore)
           say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
         end
         new_branch = options["new_branch"] || false
 
+        if options["sync"] or sync
+          new_branch_exp = @project.get_new_branch
+          if new_branch_exp
+            new_branch = new_branch_exp
+          end
+        end
         result = @project.compare_idx(new_branch)
         commit = result["result"]["commit"]
         if !link
@@ -749,9 +1657,23 @@ module Cnvrg
           say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
         end
         result = result["result"]["tree"]
+        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
+        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
+        #   if all.size == 1
+        #     num = "conflict"
+        #   else
+        #     num =  "conflicts"
+        #   end
+        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
+        #   say "#{all.join("\n")}"
+        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
+        #   exit(1)
+        #
+        # end
+        check = Helpers.checkmark()
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_end(0)
-          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
+          say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
           return true
         end
         update_count = 0
@@ -765,7 +1687,7 @@ module Cnvrg
             say "Updating #{update_total} files", Thor::Shell::Color::BLUE
           end
         else
-          say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
+          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
 
         end
 
@@ -799,21 +1721,21 @@ module Cnvrg
           end
 
           # delete
-          result["deleted"].each do |f|
+          deleted = update_deleted(result["deleted"])
+          deleted.each do |f|
             relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
             if relative_path.end_with?("/")
               if @files.delete_dir(f, relative_path, commit_sha1)
-                update_count += 1
+                # update_count += 1
                 successful_updates<< relative_path
               end
             else
               if @files.delete_file(f, relative_path, commit_sha1)
-                update_count += 1
+                # update_count += 1
                 successful_updates<< relative_path
               end
             end
           end
-          log_end(0)
 
         rescue SignalException
           log_end(-1)
@@ -821,11 +1743,14 @@ module Cnvrg
           say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
           exit(0)
         rescue => e
-          log_end(1, e.message)
+          log_end(-1, e.message)
           @files.rollback_commit(commit_sha1)
           say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
           exit(0)
         end
+        if !result["deleted"].nil? and !result["deleted"].empty?
+          update_count += result["deleted"].size
+        end
         if update_count == update_total
           res = @files.end_commit(commit_sha1)
           if (Cnvrg::CLI.is_response_success(res, false))
@@ -835,7 +1760,7 @@ module Cnvrg
 
               @project.update_idx_with_commit!(commit_sha1)
             rescue => e
-              log_end(1, e.message)
+              log_end(-1, e.message)
               @files.rollback_commit(commit_sha1)
               say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
               exit(1)
@@ -845,7 +1770,6 @@ module Cnvrg
             if image and image.is_docker
               image.update_image_activity(commit_sha1, nil)
             end
-            check = Helpers.checkmark()
 
             if options["verbose"]
               say "#{check} Done", Thor::Shell::Color::BLUE
@@ -861,7 +1785,13 @@ module Cnvrg
               end
               say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
             else
-              say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN unless options["sync"]
+              if (options["sync"] or sync) and direct
+                say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
+
+              else
+                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
+              end
 
             end
 
@@ -873,13 +1803,14 @@ module Cnvrg
           end
         else
           log_end(1, "error. Rolling Back all changes")
+          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
 
           @files.rollback_commit(commit_sha1)
         end
-      rescue
+      rescue => e
         log_end(-1)
 
-        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
         exit(1)
       rescue SignalException
@@ -894,27 +1825,57 @@ module Cnvrg
     end
 
     desc 'download', 'Download updated files'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
 
-    def download
+    def download(sync=false, ignore_list="")
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
         project_home = get_project_home
         @project = Project.new(project_home)
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        ignore = options[:ignore] || ""
+        if ignore.nil? or ignore.empty?
+          ignore = ignore_list
+        end
+        data_ignore = data_dir_include()
+        if !data_ignore.nil?
+          if ignore.nil? or ignore.empty?
+            ignore = data_ignore
+          else
+            ignore ="#{ignore},#{data_ignore}"
+          end
+        end
+        if !@project.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
         new_branch = options["new_branch"] || false
 
         res = @project.compare_idx(new_branch)["result"]
+
         result = res["tree"]
+
         commit = res["commit"]
-        if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
-          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
+        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
           log_end(0)
           return true
         end
+        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
+        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
+        #   if all.size == 1
+        #     num = "1 conflict"
+        #   else
+        #     num =  "#{result["conflicts"].size} conflicts"
+        #   end
+        #   say "Project contains #{num}:", Thor::Shell::Color::RED
+        #   say "#{all.join("\n")}"
+        #   say "Please fix them, and retry", Thor::Shell::Color::RED
+        #   exit(1)
+        #   end
         update_count = 0
         update_total = result["updated_on_server"].size + result["conflicts"].size
 
@@ -936,7 +1897,7 @@ module Cnvrg
 
         result["conflicts"].each do |f|
           relative_path = f.gsub(/^#{@project.local_path}/, "")
-          if @files.download_file_s3(f, relative_path, project_home, conflict=true)
+          if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
             successful_changes << relative_path
           end
 
@@ -967,12 +1928,35 @@ module Cnvrg
             say successful_changes.join("\n"), Thor::Shell::Color::GREEN
             say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
           else
-            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
+            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
           end
 
 
           log_end(0)
         end
+      rescue => e
+        log_end(-1)
+
+        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+        if successful_changes.nil?
+          exit(1)
+        end
+        successful_changes.each do |f|
+
+          abs_path = "#{@project.local_path}/#{f}"
+          filename = File.basename abs_path
+          say "revoking #{filename}"
+          if result["conflicts"].include? f
+            @files.revoke_download_file(abs_path, f, filename, true)
+          elsif result["updated_on_server"].include? f
+            if File.directory? abs_path
+              @files.revoke_download_dir(abs_path, f, project_home)
+            else
+              @files.revoke_download_file(project_home, abs_path, filename)
+            end
+          end
+        end
+        exit(1)
       rescue SignalException
         log_end(-1)
         say "\nAborting", Thor::Shell::Color::BLUE
@@ -1026,6 +2010,7 @@ module Cnvrg
         if !response["result"]["tree"].nil?
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
           File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          current_tree = Dir.entries(".").reject { |file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" }
 
           response["result"]["tree"].each do |f|
             relative_path = f[0].gsub(/^#{@project.local_path}/, "")
@@ -1037,7 +2022,7 @@ module Cnvrg
               end
             else
               # blob
-              if @files.download_file_s3(f[0], relative_path, project_home)
+              if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=commit_sha1)
                 current_tree.delete(relative_path)
 
                 successful_changes << relative_path
@@ -1055,43 +2040,100 @@ module Cnvrg
     end
 
 
+    desc 'data_jump', 'jump to specific commit', :hide => true
+
+    def data_jump(*commit_sha1)
+      begin
+        verify_logged_in()
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        if commit_sha1.nil? or commit_sha1.empty?
+          commit_sha1 = @dataset.last_local_commit
+        end
+        response = @dataset.compare_commits(commit_sha1)
+        successful_changes = []
+        if !response["result"]["status"].nil?
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          status = response["result"]["status"]
+          (status["delete"]).each do |f|
+            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
+            FileUtils.rm_rf(relative_path)
+          end
+          # current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
+          (status["dirs"]).each do |f|
+            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
+            # dir
+            if @files.download_dir(dataset_dir, relative_path)
+              # current_tree.delete(relative_path[0, relative_path.size-1])
+              successful_changes << relative_path
+            end
+          end
+          (status["download"]).each do |f|
+            relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
+            # dir
+            if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
+              successful_changes << relative_path
+            end
+          end
+
+
+          say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
+          log_end(0)
+        end
+
+      rescue SignalException
+        log_end(-1)
+        exi(1)
+      end
+    end
+
     desc 'sync', 'Sync with remote server'
-    method_option :new_branch, :type => :boolean, :aliases => ["--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
 
-    def sync
+    def sync(direct=true)
+      verify_logged_in(true) if direct
       if options["verbose"]
-      say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
       else
         say 'Syncing project', Thor::Shell::Color::BLUE
       end
 
 
-      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
-      invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
-      check = Helpers.checkmark()
+      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
+      invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+             :ignore => options[:ignore]
 
-      say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
 
     end
 
 
     desc 'run cmd', 'Runs an experiment'
-    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
-    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
+    method_option :local, :type => :boolean, :aliases => ["-l", "--local"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
-    method_option :sync_before, :type => :boolean, :aliases => ["--sb"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["--sa"], :default => true
-    method_option :title, :type => :string, :aliases => ["--t"], :default => ""
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
     method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["--en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["--uo"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["--c"], :default => ""
-    method_option :schedule, :type => :string, :aliases => ["--s"], :default => ""
-    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
 
     def run(*cmd)
       sync_before = options["sync_before"]
@@ -1104,38 +2146,73 @@ module Cnvrg
       local = options["local"]
       schedule = options["schedule"]
       image = options["image"]
+      grid = options["grid"]
+      data = options["data"]
+      data_commit = options["data_commit"]
+      ignore = options["ignore"]
+      options_hash = Hash[options]
+      real_options = []
+      options_hash.each do |o|
+        real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu", "gpuxl", "gpuxxl"].include? o[0])
+      end
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-               :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
+               :log => log, :email_notification => email_notification, :upload_output => upload_output,
+               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
         return
       else
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
+        real_options.delete(["local", false])
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
+        if !instance_type.nil? and !instance_type.empty?
+          real_options << ["machine_type", instance_type]
+        end
+        exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+        cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image
+               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+               :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
         return
       end
-    end
 
+      # if local
+      #
+      # else
 
-    desc '', ''
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
-    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
-    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
-    method_option :indocker, :type => :boolean,  :default => false
 
+      #   invoke :exec_remote, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
+      #          :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+      #          :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
+      #   return
+      # end
+    end
+
+
+    desc '', '', :hide => true
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb,--sync_before"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa,--sync_after"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en,--email_notification"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["-uo,--upload_output"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :indocker, :type => :boolean, :aliases => ["--indocker"], :default => false
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
+    method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     def exec(*cmd)
+
       log = []
       cpu_average =0
       memory_average = 0
       verify_logged_in(true)
       log_start(__method__, args, options)
-      working_dir = Dir.pwd
+      working_dir = is_cnvrg_dir
+      script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
 
 
       sync_before = options["sync_before"]
@@ -1143,11 +2220,14 @@ module Cnvrg
       print_log = options["log"]
       title = options["title"]
       commit = options["commit"] || nil
-      image = options["image"]  || nil
-      indocker = options["indocker"]  || false
+      image = options["image"] || nil
+      indocker = options["indocker"] || false
+      ignore = options[:ignore] || ""
+
 
       email_notification = options["email_notification"]
       upload_output = options["upload_output"]
+      upload_output = "1m" if upload_output.nil? or upload_output.empty?
       time_to_upload = calc_output_time(upload_output)
       project_home = get_project_home
       @project = Project.new(project_home)
@@ -1159,7 +2239,8 @@ module Cnvrg
         else
           if sync_before
             # Sync before run
-            invoke :sync, [], :new_branch => is_new_branch
+
+            invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
           end
         end
         #set image for the project
@@ -1167,40 +2248,46 @@ module Cnvrg
           invoke :set_image, [image]
         end
         if !indocker
-        image_proj = is_project_with_docker(working_dir)
+          image_proj = is_project_with_docker(working_dir)
 
 
-        if image_proj and image_proj.is_docker
-          container= image_proj.get_container
-          if !container
-            say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
-            exit(1)
-          end
+          if image_proj and image_proj.is_docker
+            container= image_proj.get_container
+            if !container
+              say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
+              exit(1)
+            end
 
 
-          exec_args = args.flatten.join(" ")
-          options_hash = Hash[options]
-          options_hash.except!("image", "indocker")
-          exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-          command_to_run = cmd.join(" ")
-          command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
-          puts container.exec(command, tty: true)
-          container.stop()
-          exit(0)
+            exec_args = args.flatten.join(" ")
+            options_hash = Hash[options]
+            options_hash.except!("image", "indocker")
+            exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+            command_to_run = copy_args.join(" ")
+            command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
+            puts container.exec(command, tty: true)
+            container.stop()
+            exit(0)
+          end
         end
+        remote = options["remote"]
+        if remote
+          docker_id = `cat /etc/hostname`
+          docker_id = docker_id.strip()
         end
+        is_on_gpu = options["gpu"]
         start_commit = @project.last_local_commit
         cmd = cmd.join("\s")
 
-        say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
-
         @exp = Experiment.new(@project.owner, @project.slug)
 
         platform = RUBY_PLATFORM
         machine_name = Socket.gethostname
         begin
-          machine_activity = @exp.get_machine_activity(Dir.pwd)
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          machine_activity = @exp.get_machine_activity(working_dir)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
+          say "Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
+          say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -1212,19 +2299,34 @@ module Cnvrg
                 begin
                   stdout.each do |line|
                     cur_time = Time.now
-                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
-                    monitor_by = monitor.split(" ")
-                    memory = monitor_by[2]
-                    cpu = monitor_by[1]
+                    if remote
+                      stats = usage_metrics_in_docker(docker_id)
+                      cpu = stats[0]
+                      memory = stats[1]
+                    else
+                      memory = memory_usage()
+                      cpu = cpu_usage()
+                    end
+
                     memory_total << memory.to_f
                     cpu_total << cpu.to_f
                     real_time= Time.now-real
 
-                    cur_log = {time: cur_time,
+                    cur_log = { time: cur_time,
                                message: line,
                                type: "stdout",
-                               real: real_time}
-                    if print_log
+                               real: real_time,
+                                cpu: cpu,
+                                memory: memory
+                    }
+                    if is_on_gpu
+                      gpu_stats = gpu_util
+                      gpu_utilization = gpu_stats[0]
+                      gpu_memory_util = gpu_stats[1]
+                      cur_log.merge!(gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
+                    end
+
+                      if print_log
                       puts cur_log
                     end
                     log << cur_log
@@ -1233,8 +2335,8 @@ module Cnvrg
                       if time_to_upload !=0
                         if time_to_upload <= Time.now - start_loop
                           #upload current log
-                          cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
-                          memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+                          # cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                          # memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
 
                           @exp.upload_temp_log(log, cpu_average, memory_average)
                           log = []
@@ -1258,8 +2360,7 @@ module Cnvrg
                   end
 
                 rescue Errno::EIO => e
-                  puts e
-                    # break
+                  # break
                 rescue Errno::ENOENT
                   log_end(1, "command #{cmd} isn't valid")
 
@@ -1271,8 +2372,11 @@ module Cnvrg
                   exp_success = false
                   say "The process exited!", Thor::Shell::Color::RED
                 rescue => e
-                  puts e
-                  log_end(1, e.message)
+                  log_end(-1, e.message)
+                  res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
+
+                  say "Error occurred,aborting", Thor::Shell::Color::RED
+                  exit(0)
                 end
                 ::Process.wait pid
                 cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
@@ -1283,7 +2387,7 @@ module Cnvrg
                 end
                 if !exp_success
                   if !Cnvrg::Helpers.internet_connection?
-                    wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+                    wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
                     if wait_offline
                       say "Waiting until your'e online..", Thor::Shell::Color::BLUE
                       while !Cnvrg::Helpers.internet_connection?
@@ -1306,23 +2410,21 @@ module Cnvrg
                 end
                 if sync_after
                   # Sync after run
-                  if !commit.nil?
-                    invoke :sync, [], :new_branch => true
 
-                  else
-                    invoke :sync, [], :new_branch => true
-                  end
+
+                  download(sync=true, ignore_list=ignore)
+                  upload(link=false, sync=true, direct=false, ignore_list=ignore)
+
                 end
                 end_commit = @project.last_local_commit
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                 check = Helpers.checkmark()
-                say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
+                say "#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN
                 log_end(0)
               end
             rescue => e
-              puts e
-              log_end(1, e.message)
+              log_end(-1, e.message)
               if container
                 container.stop()
               end
@@ -1349,7 +2451,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
@@ -1388,7 +2490,7 @@ module Cnvrg
         machine_name = Socket.gethostname
         begin
 
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -1458,7 +2560,9 @@ module Cnvrg
                   exp_success = false
                   say "The process exited!", Thor::Shell::Color::RED
                 rescue => e
-                  log_end(1, e.message)
+                  log_end(-1, e.message)
+                  say "Error occurred, aborting", Thor::Shell::Color::RED
+                  exit(0)
                 end
                 ::Process.wait pid
                 cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
@@ -1508,7 +2612,7 @@ module Cnvrg
                 log_end(0)
               end
             rescue => e
-              log_end(1, e.message)
+              log_end(-1, e.message)
               say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
               exit(1)
             end
@@ -1532,23 +2636,33 @@ module Cnvrg
       end
     end
 
-    desc '', ''
-    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
-    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
-    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
-    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
-    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
-    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
-    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => nil
-    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
+    desc '', '', :hide => true
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
+    method_option :machine_type, :type => :string, :default => ""
+    method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
 
     def exec_remote(*cmd)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
+      path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
+
       begin
+        grid = options["grid"] || nil
+        data = options["data"] || nil
+        data_commit = options["data_commit"] || nil
+        sync_before = options["sync_before"]
 
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
@@ -1570,10 +2684,22 @@ module Cnvrg
           new_time = new_time[0, new_time.size-6] #remove timezone
           schedule = "at #{new_time}"
         end
+        upload_output = options["upload_output"]
+        time_to_upload = calc_output_time(upload_output)
+        if time_to_upload==0 or time_to_upload==-1
+          upload_output_option = "--upload_output=1m"
+        else
+          upload_output_option = "--upload_output=#{upload_output}"
+        end
+        remote = "--remote=true"
+        if !instance_type.nil? and instance_type.include? "gpu"
+          remote= "#{remote} --gpu=true"
+        end
+
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options} --uo=1m #{cmd.flatten.join(" ")}"
+        command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -1584,19 +2710,20 @@ module Cnvrg
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image,[choose_image]
+          invoke :set_image, [choose_image]
         end
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
           # say "Couldn't find image related to project", Thor::Shell::Color::RED
-          default_image_name = "cnvrg"
+
+          image_slug = "cnvrg"
           if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
-            default_image_name = "cnvrg-gpu"
+            image_slug = "cnvrg_gpu"
           end
           # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
           # if default
-            image = Images.new(working_dir, default_image_name)
-            image_slug = image.image_slug
+          #   image = Images.new(working_dir, default_image_name)
+          #   image_slug = image.image_slug
           # else
           #   exit(0)
           # end
@@ -1604,16 +2731,27 @@ module Cnvrg
           image_slug = image.image_slug
         end
 
+        invoke :sync, [false], []  if sync_before
+
 
+        if command.include? "'"
+          oc = command.to_enum(:scan, /'/).map { Regexp.last_match }
+          pairs = oc.enum_for(:each_slice, 2).to_a
+          pairs.each_with_index do |p,i|
+            add=0
+            if i!=0
+              add=2*i
+            end
+            total_loc = command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add]
+            command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add] = "\"#{total_loc}\""
+          end
 
-        invoke :sync, [], []
 
+        end
         say "Running remote experiment", Thor::Shell::Color::BLUE
         exp = Experiment.new(project.owner, project.slug)
-
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
-
           # if res["result"]["machine"] == -1
           #   say "There are no available machines", Thor::Shell::Color::BLUE
           #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
@@ -1645,7 +2783,13 @@ module Cnvrg
           #   end
           # else
           check = Helpers.checkmark()
-          say "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}", Thor::Shell::Color::GREEN
+          str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
+
+          if res["result"]["grid"]
+            str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
+          end
+
+          say str, Thor::Shell::Color::GREEN
 
           exit(0)
           # end
@@ -1665,21 +2809,30 @@ module Cnvrg
     end
 
     desc 'deploy', 'Deploys model to production'
-    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
-    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
-    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :schedule, :type => :string, :aliases => ["--schedule", "-s"], :default => ""
+
+    method_option :commit, :type => :string, :aliases => ["--commit", "-c"], :default => ""
+    method_option :workers, :type => :string, :aliases => ["--workers", "-w"], :default => ""
+    method_option :file_as_input, :type => :boolean, :aliases => ["--input", "-i"], :default => false
 
     def deploy(file_to_run, function)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
       begin
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+        instance_type = get_instance_type(instances)
 
-        instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
 
 
-
         if !schedule.nil? and !schedule.empty?
 
           local_timestamp = get_schedule_date
@@ -1688,26 +2841,34 @@ module Cnvrg
         project = Project.new(working_dir)
         commit_to_run = options["commit"] || nil
 
+        workers = options["workers"] || nil
+        begin
+          num_workers = workers.to_i
+        rescue
+          say "Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED
+          exit(1)
+        end
+        file_as_input = options["file_as_input"] || false
+
 
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
-          say "Couldn't find image related to project", Thor::Shell::Color::RED
-          default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
-          if default
-            image = Images.new(working_dir, "cnvrg")
-            image_slug = image.image_slug
-          else
-            exit(0)
-          end
+          # say "Couldn't find image related to project", Thor::Shell::Color::RED
+          # default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
+          # if default
+          image = Images.new(working_dir, "cnvrg")
+          image_slug = image.image_slug
+          # else
+          #   exit(0)
+          # end
         else
           image_slug = image.image_slug
         end
 
 
+        invoke :sync, [false], []
 
-        invoke :sync, [], []
-
-        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
+        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input)
 
         if Cnvrg::CLI.is_response_success(res)
 
@@ -1763,27 +2924,37 @@ module Cnvrg
 
     method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
-    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
-    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
+    method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
-    desc 'starts a notebook session', 'starts a notebook session remotely or locally'
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+
 
-    def notebook()
+    desc 'notebook', 'starts a notebook session remotely or locally'
+
+    def notebook
       local = options["local"]
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
-
+      image = options["image"]
+      data = options["data"]
+      data_commit = options["data_commit"]
       if local
-        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel
+        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
         return
       else
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
 
-        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type
+        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
+               :data => data, :data_commit => data_commit
         return
 
       end
@@ -1792,10 +2963,12 @@ module Cnvrg
     end
 
     desc 'remote_notebook', 'run notebook server on remote server'
-    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
-    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
-    method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
-    method_option :image, :type => :string, :aliases => ["--i"], :default => ""
+    method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :machine_type, :type => :string, :default => ""
+    method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
+    method_option :image, :type => :string, :aliases => ["-i"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
 
 
     def remote_notebook()
@@ -1807,13 +2980,15 @@ module Cnvrg
       notebook_dir = options["notebook_dir"]
       instance_type = options["machine_type"] || nil
       kernel = options["kernel"] || nil
+      data = options["data"]
+      data_commit = options["data_commit"]
 
 
       begin
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image,[choose_image]
+          invoke :set_image, [choose_image]
         end
         @image = is_project_with_docker(working_dir)
         if !@image or !@image.is_docker
@@ -1824,17 +2999,16 @@ module Cnvrg
           end
           # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
           # if default
-            @image = Images.new(working_dir, default_image_name)
+          @image = Images.new(working_dir, default_image_name)
           # else
           #   exit(0)
           # end
         end
 
-        invoke :sync, [], []
-
+        invoke :sync, [false], []
 
 
-        res = @image.remote_notebook(notebook_dir, instance_type, kernel)
+        res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
           if res["result"]["machine"] == -1
             say "There are no available machines", Thor::Shell::Color::BLUE
@@ -1873,7 +3047,7 @@ module Cnvrg
             note_url = res["result"]["notebook_url"]
             @image.set_note_url(note_url)
             check = Helpers.checkmark()
-            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
+            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
             # Launchy.open(url)
 
             exit(0)
@@ -1941,9 +3115,8 @@ module Cnvrg
 
         end
       rescue => e
-        puts e
-        log_end(-1)
-        say "Error occurd, Aborting"
+        log_end(-1, e.message)
+        say "Error occurred, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
@@ -2004,9 +3177,8 @@ module Cnvrg
           end
         end
       rescue => e
-        puts e
-        log_end(-1)
-        say "Error occurd, Aborting"
+        log_end(-1, e.message)
+        say "Error occurred, aborting"
         if container
           container.stop()
         end
@@ -2023,6 +3195,7 @@ module Cnvrg
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :image, :type => :string, :aliases => ["-i"], :default => ""
 
     def run_notebook
 
@@ -2036,7 +3209,7 @@ module Cnvrg
         notebook_dir = options["notebook_dir"]
         remote = options["remote"] || false
         kernel = options["kernel"] || ""
-
+        notebooks_pid = nil
 
         if notebook_dir.empty?
           notebook_dir = project_dir
@@ -2044,107 +3217,148 @@ module Cnvrg
 
           notebook_dir = project_dir+ notebook_dir
         end
-        say "Linking notebook directory to: #{File.basename notebook_dir}", Thor::Shell::Color::BLUE
-        image = is_project_with_docker(Dir.pwd)
-        if image and image.is_docker and !remote
-          container= image.get_container
-          if !container
+        choose_image = options["image"]
 
-            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+        if !choose_image.nil? and !choose_image.empty?
+          invoke :set_image, [choose_image]
+        end
+
+        image = is_project_with_docker(Dir.pwd)
+        if !image
+          jupyter_installed = `which jupyter`
+          if !$?.success?
+            say "Could not find jupyter, Is it installed?", Thor::Shell::Color::RED
             exit(1)
           end
-        else
-          say "Project is not configured with any image", Thor::Shell::Color::RED
-          exit(1)
 
-        end
-        if options["verbose"]
-          say "Syncing project before running", Thor::Shell::Color::BLUE
-          say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
-        end
-        @project = Project.new(project_dir)
 
-        start_commit = @project.last_local_commit
+          cmd = "jupyter-notebook --port=8888"
+          PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
+            begin
+              notebooks_pid = pid
+              stdout.each do |line|
+                puts line
+
+              end
+
+            rescue Errno::EIO => e
+              # break
+            rescue Errno::ENOENT
+              log_end(1, "command #{cmd} isn't valid")
+
+
+              say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+            rescue PTY::ChildExited
+              log_end(1, "proccess exited")
+              say "The process exited!", Thor::Shell::Color::RED
+            rescue => e
+              log_end(-1, e.message)
+              say "Error occurred,aborting", Thor::Shell::Color::RED
+              exit(0)
+
+            end
+
 
-        if (note_slug=image.note_slug)
-          say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
-          new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
-          if !new
-            exit(0)
           end
 
         end
-        invoke :sync, [], :verbose => options["verbose"]
-        say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
-        #replace url
-        base_url = get_base_url()
 
-        local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
-        command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
-        container.exec(command, tty: true)
-        container.stop()
-        container.start()
+        if image and image.is_docker and !remote
+          container= image.get_container
+          if !container
+            say "Couldn't start docker container", Thor::Shell::Color::RED
+            exit(1)
 
-        @note = Experiment.new(@project.owner, @project.slug)
-        port = image.container_port()
+          end
 
-        command = ["/bin/bash", "-lc", "jupyter notebook list"]
-        list = container.exec(command, tty: true)[0]
-        if list.empty? or list.nil?
-          say "Couldn't start notebook server", Thor::Shell::Color::RED
-          log_end(1, "can't start notebook server")
-          exit(1)
-        end
+          if options["verbose"]
+            say "Syncing project before running", Thor::Shell::Color::BLUE
+            say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+          end
+          @project = Project.new(project_dir)
+
+          start_commit = @project.last_local_commit
+
+          if (note_slug=image.note_slug)
+            say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
+            new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
+            if !new
+              exit(0)
+            end
+
+          end
+          invoke :sync, [false], :verbose => options["verbose"]
+          say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
+          #replace url
+          base_url = get_base_url()
+
+          local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
+          command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+          container.exec(command, tty: true)
+          container.stop()
+          container.start()
+          sleep(7)
+          @note = Experiment.new(@project.owner, @project.slug)
+          port = image.container_port()
+
+          command = ["/bin/bash", "-lc", "jupyter notebook list"]
+          list = container.exec(command, tty: true)[0]
+          if list.empty? or list.nil?
+            say "Couldn't start notebook server", Thor::Shell::Color::RED
+            log_end(1, "can't start notebook server")
+            exit(1)
+          end
 
-        url =  list[list.size-1].split(" ::")
-        token = url[0][(url[0].index("=")+1)..-1]
+          result = ""
+          list.each do |r|
+            if r.include? "http"
+              result = r
+            end
+          end
+          token = result.to_s.split("::")[0].to_s.match(/(token=)(.+)\s/)[2]
 
-        # machine_activity = @note.get_machine_activity(project_dir)
+          # machine_activity = @note.get_machine_activity(project_dir)
 
 
-        slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
-        image.set_note_url(slug)
-        note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
+          slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
+          image.set_note_url(slug)
+          note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
 
 
-        if !note_url.empty?
-          check = Helpers.checkmark()
+          if !note_url.empty?
+            check = Helpers.checkmark()
 
-          say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
-          Launchy.open(note_url)
-        else
-          say "Couldn't start notebook server", Thor::Shell::Color::RED
-          log_end(1, "can't start notebook server")
-          exit(1)
+            say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
+            Launchy.open(note_url)
+          else
+            say "Couldn't start notebook server", Thor::Shell::Color::RED
+            log_end(1, "can't start notebook server")
+            exit(1)
+          end
         end
 
-          # jup =verify_software_installed("jupyter-notebook")
-          # logs = `#{jup} --no-browser --ip=0.0.0.0  --notebook-dir=#{notebook_dir}`
-          # url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
-          # if !url.empty?
-          #   check = Helpers.checkmark()
-          #
-          #   say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
-          #   log_end(0)
-          # else
-          #   say "Couldn't start notebook server", Thor::Shell::Color::RED
-          #   log_end(1, "can't start notebook server")
-          #   exit(1)
-          # end
+
       rescue => e
-        log_end(-1)
-        puts e
+        log_end(-1, e.message)
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
       rescue SignalException
-        log_end(-1)
-        if container
-          container.stop()
+        if !notebooks_pid.nil?
+          ::Process.kill(0, notebooks_pid)
+          say "#{check} Notebook has stopped successfully", Thor::Shell::Color::GREEN
+
+
+          invoke :sync, [false], []
+        else
+          log_end(-1)
+
+          if container
+            container.stop()
+          end
         end
-        say "Aborting"
-        exit(1)
+
       end
 
 
@@ -2165,7 +3379,7 @@ module Cnvrg
 
 
         say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
-        invoke :sync, [], :verbose => options["verbose"]
+        invoke :sync, [false], :verbose => options["verbose"]
 
         say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
 
@@ -2210,8 +3424,7 @@ module Cnvrg
           exit(0)
         end
       rescue => e
-        puts e
-        log_end(-1)
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -2261,8 +3474,7 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1)
-        puts e
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -2325,14 +3537,12 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1)
-        puts e
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
         end
       rescue SignalException
-        log_End(-1)
         if container
           container.stop()
         end
@@ -2341,7 +3551,100 @@ module Cnvrg
       end
 
     end
-    desc 'run commands inside containers', 'run commands inside containers'
+
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
+    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
+    method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
+    method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
+    method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
+    method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
+
+    desc 'create_custom_image', 'run commands inside containers', :hide=>true
+
+    def create_custom_image(image_name)
+      begin
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+                   "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
+      instance_type = get_instance_type(instances)
+      image_extend = options["image"]
+      public = options["public"]
+      base = options["base"]
+      python3 = options["python3"]
+      owner = CLI.get_owner
+      checks = Helpers.checkmark()
+
+      say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
+
+      resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
+      if Cnvrg::CLI.is_response_success(resp,false)
+        image_slug = resp["result"]["slug"]
+        container = resp["result"]["machine_c"]
+        say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
+        say "Connecting to machine", Thor::Shell::Color::BLUE
+        ssh = Ssh.new(resp)
+        if !ssh.is_ssh
+          say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
+          Images.revoke_custom_new_image(owner, image_slug)
+          log_end(-1,"Couldn't connect to machine,aborting")
+        end
+        say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
+        begin
+
+          while true
+            command = ask("$>")
+            puts ssh.exec_command(command)
+          end
+
+        rescue SignalException
+          say "Commiting Image..", Thor::Shell::Color::BLUE
+
+        end
+        resp = Images.commit_custom_image(owner,image_slug)
+        if Cnvrg::CLI.is_response_success(resp,false)
+          say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
+        end
+        if ssh
+          ssh.close_ssh()
+        end
+
+
+
+
+
+
+
+      end
+      rescue =>e
+        puts e
+        if image_slug
+          Images.revoke_custom_new_image(owner, image_slug)
+        end
+        if ssh
+          ssh.close_ssh()
+        end
+
+
+      rescue SignalException
+        if image_slug
+          Images.revoke_custom_new_image(owner, image_slug)
+        end
+        if ssh
+          ssh.close_ssh
+        end
+        say "\nAborting"
+        exit(1)
+      end
+
+    end
+
+
+    desc 'build', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
     def build(*cmd)
@@ -2352,10 +3655,10 @@ module Cnvrg
         working_dir = is_cnvrg_dir
         install_file = options["install"] || nil
         if !install_file.nil?
-          commands = File.open(install_file).read.chop.gsub!("\n",",").split(",")
+          commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
 
         else
-          commands  = [cmd.join(" ")]
+          commands = [cmd.join(" ")]
         end
 
 
@@ -2373,25 +3676,25 @@ module Cnvrg
 
         end
         commands.each do |c|
-        if c.include? "pip"
-          c.sub("pip","/opt/ds/bin/pip")
-        end
-        if c.include? "pip3"
-          c.sub("pip3","/opt/ds3/bin/pip3")
-        end
+          if c.include? "pip"
+            c.sub("pip", "/opt/ds/bin/pip")
+          end
+          if c.include? "pip3"
+            c.sub("pip3", "/opt/ds3/bin/pip3")
+          end
 
-        say "Running #{c}", Thor::Shell::Color::BLUE
-        command = ["/bin/bash", "-lc", "#{c}"]
-        res = container.exec(command, tty: false)
-        if res[2] != 0
-         say  "Could not run command: #{c}, #{res[1][0]}" , Thor::Shell::Color::RED
-          container.stop()
-          log_end(0)
+          say "Running #{c}", Thor::Shell::Color::BLUE
+          command = ["/bin/bash", "-lc", "#{c}"]
+          res = container.exec(command, tty: false)
+          if res[2] != 0
+            say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
+            container.stop()
+            log_end(0)
 
-          exit(1)
-        end
-        say res[0].join("\n")
-        image.store_image_build_commands(working_dir,c)
+            exit(1)
+          end
+          say res[0].join("\n")
+          image.store_image_build_commands(working_dir, c)
         end
 
         checks = Helpers.checkmark()
@@ -2402,8 +3705,7 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1)
-        puts e
+        log_end(-1, e.message)
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
@@ -2452,8 +3754,9 @@ module Cnvrg
         log_end(0)
         return new_image.id
       rescue => e
-        puts e
-
+        log_end(-1, e.message)
+        say "\nError occurred,aborting"
+        exit(1)
       rescue SignalException
         log_end(-1)
         say "\nAborting"
@@ -2496,7 +3799,7 @@ module Cnvrg
 
     end
 
-    desc 'push image to cnvrg repository', 'push image to cnvrg repository'
+    desc 'push', 'push image to cnvrg repository'
 
     def push(*name)
       verify_logged_in(true)
@@ -2514,22 +3817,24 @@ module Cnvrg
         end
         if !name.nil? and !name.empty?
           if name.include? " "
-            name.gsub!(" ","_")
+            name.gsub!(" ", "_")
           end
         end
-        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n",",")
+        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
         if stored_commands.nil? or stored_commands.empty?
           say "Nothing to push", Thor::Shell::Color::BLUE
           exit(0)
         end
 
         say "Pushing new image", Thor::Shell::Color::BLUE
-        if image.create_custom_image(name,working_dir,stored_commands)
+        if image.create_custom_image(name, working_dir, stored_commands)
 
           say "Image was updated successfully", Thor::Shell::Color::GREEN
         end
       rescue => e
-        puts e
+        log_end(-1, e.message)
+        say "error occurred, aborting"
+
       end
     end
 
@@ -2591,8 +3896,8 @@ module Cnvrg
           exit(1)
         end
       rescue => e
-        puts e
-
+        log_end(-1, e.message)
+        say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
       rescue SignalException
         log_end(-1)
 
@@ -2601,7 +3906,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def upload_log()
       log_path = '/home/ds/app/uwsgi.log'
@@ -2610,28 +3915,32 @@ module Cnvrg
 
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def exec_container(container_id, *cmd)
       container = Docker::Container.get(container_id)
       container.start()
       cnvrg_command = cmd.join(" ")
       command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
-      res = container.exec(command, tty: true,wait:5400)[0]
+      res = container.exec(command, tty: true, wait: 5400)[0]
       say res
-
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def port_container(container_id)
       container = Docker::Container.get(container_id)
       say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
+    end
 
+    desc '', '', :hide => true
 
+    def tensor_port_container(container_id)
+      container = Docker::Container.get(container_id)
+      say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def stop_container(container_id)
       container = Docker::Container.get(container_id)
@@ -2640,13 +3949,13 @@ module Cnvrg
 
     end
 
-    desc '', ''
-    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+    method_option :app_dir, :type => :string, :aliases => ["-d"], :default => "/home/ds/notebooks"
+    method_option :cmd, :type => :string, :aliases => ["-c"], :default => "/usr/local/cnvrg/run_ipython.sh"
 
 
-    def config_remote(image_name, port=7654)
+    def config_remote(image_name, port=7654, tensport=6006)
       local_images = Docker::Image.all
 
       docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -2659,20 +3968,25 @@ module Cnvrg
         login_content = options["login"]
         app_dir = options["app_dir"]
         cmd= options["cmd"]
+        volume_from = options["volume"]
 
         image_settings = {
             'Image' => "#{image_name}:latest",
-            'User' => 'ds',
+
             'Cmd' => cmd,
             'WorkingDir' => app_dir,
             'ExposedPorts' => {
                 '8888/tcp' => {},
             },
             'HostConfig' => {
+                'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker" ],
                 'PortBindings' => {
                     '8888/tcp' => [
                         {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
                     ],
+                    '6006/tcp' => [
+                        {'HostPort' => "#{tensport}", 'HostIp' => 'localhost'}
+                    ],
                 },
             },
         }
@@ -2680,21 +3994,104 @@ module Cnvrg
         container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        # container.exec(command, tty: true)
+        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        # container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        say "#{container.id}:#{port}##{tensport}"
+      rescue => e
+        puts e
+        if e.message.include? "is not running"
+          return config_remote(image_name, port-1, tensport-1)
+        end
+
+        if container
+          container.kill()
+        end
+        return false
+      end
+    end
+
+
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+
+    def config_netrc(container)
+
+      login_content = options["login"]
+
+      container = Docker::Container.get(container)
+      command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+      container.exec(command, tty: true)
+      command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
+      container.exec(command, tty: true)
+      command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+      container.exec(command, tty: true)
+      say "OK"
+
+    end
+
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
+    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
+    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
+
+
+    def config_remote_gpu(image_name, port=7654, tensport=6006)
+      local_images = Docker::Image.all
+
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
+      if docker_image_local.empty?
+        say "no image"
+        exit(1)
+      end
+
+      begin
+        login_content = options["login"]
+        app_dir = options["app_dir"]
+        cmd= options["cmd"]
+
+        # image_settings = {
+        #     'Image' => "#{image_name}:latest",
+        #     'User' => 'ds',
+        #     'Cmd' => cmd,
+        #     'WorkingDir' => app_dir,
+        #     'ExposedPorts' => {
+        #         '8888/tcp' => {},
+        #     },
+        #     'HostConfig' => {
+        #         'PortBindings' => {
+        #             '8888/tcp' => [
+        #                 {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
+        #             ],
+        #             '6006/tcp' => [
+        #                 {'HostPort' => "6006", 'HostIp' => 'localhost'}
+        #             ],
+        #         },
+        #     },
+        # }
+
+        container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi  -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
+        container_id = container_id.gsub("\n", "")
+        container = Docker::Container.get(container_id)
+        # container.start()
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
-        say "#{container.id}:#{port}"
+        say "#{container.id}:#{port}##{tensport}"
       rescue => e
         if e.message.include? "is not running"
-          return config_remote(image_name, port-1)
+          puts "running asgain with: #{port-1} #{tensport-1}"
+          return config_remote_gpu(image_name, port-1, tensport-1)
         end
-        puts "error"
-        puts e
+
         if container
           container.kill()
         end
@@ -2702,8 +4099,8 @@ module Cnvrg
       end
     end
 
-    desc '', ''
-    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
 
     def config_flask_remote(image_name, port=80)
       local_images = Docker::Image.all
@@ -2736,21 +4133,54 @@ module Cnvrg
         container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        say "#{container.id}:#{port}"
+      rescue => e
+        pus e
+        if e.message.include? "is not running"
+          return "port is taken"
+        end
+        puts "error"
+        if container
+          container.kill()
+        end
+        return false
+      end
+    end
+
+    desc '', '', :hide => true
+    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
+
+    def config_flask_remote_gpu(image_name, port=80)
+      local_images = Docker::Image.all
+
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
+      if docker_image_local.empty?
+        say "no image"
+        exit(1)
+      end
+
+      begin
+        login_content = options["login"]
+        container_id = `nvidia-docker run -itd -p 80:80 -w /home/ds/app #{image_name}:latest /usr/local/cnvrg/start_super.sh`
+        container_id = container_id.gsub("\n", "")
+        container = Docker::Container.get(container_id)
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
         container.exec(command, tty: true)
         command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
         container.exec(command, tty: true)
         say "#{container.id}:#{port}"
       rescue => e
+        puts e
         if e.message.include? "is not running"
           return "port is taken"
         end
         puts "error"
-        puts e
         if container
           container.kill()
         end
@@ -2758,7 +4188,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def upload_cnvrg_image(image_name)
       verify_logged_in(false)
@@ -2768,12 +4198,12 @@ module Cnvrg
 
       path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
       @files = Cnvrg::Files.new("", "")
-      python_arr = Cnvrg::Images.get_installed_packages("python")
-      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
-      system_arr = Cnvrg::Images.get_installed_packages("system")
-      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      # python_arr = Cnvrg::Images.get_installed_packages("python")
+      # py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      # system_arr = Cnvrg::Images.get_installed_packages("system")
+      # sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
       # bash_history = Cnvrg::Images.get_bash_history
-      res = @files.upload_image(path, image_name, owner, true, true, sys, py, "", "Image made by cnvrg.io team", "")
+      res = @files.upload_image(path, image_name, owner, true, true, "", "", "", "Image made by cnvrg.io team", "")
 
     end
 
@@ -2819,7 +4249,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def download_image(image_name, image_slug)
       begin
@@ -2845,8 +4275,6 @@ module Cnvrg
               end
             end
 
-
-
             return dir_path
 
           else
@@ -2880,13 +4308,13 @@ module Cnvrg
         owner = Cnvrg::CLI.get_owner()
         res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
         if Cnvrg::CLI.is_response_success(res)
-          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project",  "created by", "is_public", "last updated"
+          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
           res["result"]["images"].each do |u|
             time = Time.parse(u["created_at"])
             update_at = get_local_time(time)
             created_by = u["created_by"]
 
-            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"],  created_by, u["is_public"], update_at
+            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
           end
         end
         log_end(0)
@@ -2968,7 +4396,7 @@ module Cnvrg
 
     desc 'pull_image', 'downloads and loads an image'
 
-    def pull(image_name)
+    def pull_image(image_name)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -2981,8 +4409,18 @@ module Cnvrg
         path = download_image(image_name, image["slug"])
         if path
           say "Building image", Thor::Shell::Color::BLUE
-          Docker.options[:read_timeout]=1200
-          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile', 't' => "#{image_name}:lastest"})
+          Docker.options[:read_timeout]=216000
+          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
+            begin
+              if (log = JSON.parse(v)) && log.has_key?("stream")
+                next if log["stream"].starts_with? "Step"
+                $stdout.puts log["stream"]
+              end
+            rescue
+            end
+
+          end
+
           if not image.nil?
             FileUtils.rm_rf(path)
             checks = Helpers.checkmark()
@@ -2991,11 +4429,13 @@ module Cnvrg
             return image
             log_end(0)
           else
+
             say "Could not build image", Thor::Shell::Color::RED
             log_end(1, "Could build  image")
             return false
           end
         else
+
           say "Could not download image", Thor::Shell::Color::RED
           log_end(1, "Could build  image")
           return false
@@ -3023,7 +4463,9 @@ module Cnvrg
           #
           #   end
           # end
-      rescue
+      rescue => e
+
+
         say "Couldn't build image", Thor::Shell::Color::RED
 
       rescue SignalException
@@ -3130,7 +4572,12 @@ module Cnvrg
           if url.nil? or url.empty?
             url = "https://cnvrg.io/api"
           end
-          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url}
+          config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+          compression_path =  "#{home_dir}/.cnvrg/tmp"
+          if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
+            compression_path = config.to_h[:compression_path]
+          end
+          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
 
           File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
           return true
@@ -3184,7 +4631,7 @@ module Cnvrg
 
       def log_end(exit_status=0, error=nil)
         begin
-          if exit_status==1
+          if exit_status != 0
             $LOG.error exit_status: exit_status, error: error
           else
             $LOG.info exit_status: exit_status
@@ -3196,7 +4643,6 @@ module Cnvrg
       def self.is_response_success(response, should_exit=true)
         if response.nil?
           if !Cnvrg::Helpers.internet_connection?
-            # Cnvrg::CLI.log_end(1,"no internet connection")
             say("<%= color('Error:You seems to be offline', RED) %>")
           else
             say("<%= color('Error', RED) %>")
@@ -3231,12 +4677,26 @@ module Cnvrg
           return owner
         end
       end
+
       def get_base_url
         home_dir = File.expand_path('~')
 
         config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
         api = config.to_h[:api]
-        return api.gsub!("/api","")
+        return api.gsub!("/api", "")
+      end
+      def  get_compression_path
+        home_dir = File.expand_path('~')
+
+        config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
+        compression_path = config.to_h[:compression_path]
+        if compression_path.nil?
+          compression_path = "#{home_dir}/.cnvrg/tmp/"
+        end
+        if !compression_path.ends_with? "/"
+          compression_path = compression_path + "/"
+        end
+        return compression_path
       end
 
       def get_project_home
@@ -3290,7 +4750,6 @@ module Cnvrg
           end
           $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
         rescue => e
-          # puts e
         end
       end
 
@@ -3318,6 +4777,9 @@ module Cnvrg
 
         config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
         version_date = config.to_h[:version_last_check]
+        if version_date.nil?
+          version_date = get_start_day()
+        end
         next_day = get_start_day+ 86399
         if not (version_date..next_day).cover?(Time.now)
           if should_update_version()
@@ -3343,11 +4805,15 @@ module Cnvrg
       def is_cnvrg_dir(dir=Dir.pwd)
         current_dir = dir
         home_dir = File.expand_path('~')
+        if current_dir.eql? home_dir
+          return false
+        end
         is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
         until is_cnvrg == true
           current_dir = File.expand_path("..", current_dir)
           is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
-          if File.expand_path("..", current_dir).eql? home_dir
+          if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
+            is_cnvrg = false
             break
           end
         end
@@ -3358,6 +4824,19 @@ module Cnvrg
         end
       end
 
+      def data_dir_include()
+        all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
+        all_dirs.flatten!
+        all_dirs.each do |a|
+          if a.include? "/.cnvrg"
+            ignore = File.dirname(a)
+            return ignore
+          end
+        end
+        return nil
+
+      end
+
       def verify_software_installed(software)
         begin
           install_url = Cnvrg::CLI::INSTALLATION_URLS[software.to_sym]
@@ -3533,8 +5012,151 @@ module Cnvrg
 
       end
 
+      def update_deleted(deleted)
+        final = []
+        deleted.each do |d|
+          all_subs = d.split("/")
+          to_add = true
+          value = all_subs[0]
+          all_subs.each_with_index do |a, i|
+            if final.include? value+"/"
+              to_add = false
+              break
+            end
+            value = value+"/"+all_subs[i+1] if i <all_subs.size-1
+          end
+          final << d if to_add
+
+
+        end
+
+
+        return final
+      end
+
+      def get_cmd_path_in_dir(main_dir, sub_dir)
+        first = Pathname.new main_dir
+        second = Pathname.new sub_dir
+        relative = second.relative_path_from first
+        if relative.eql? "."
+          return ""
+        else
+          return relative
+        end
+      end
+
+      def create_tar(path_in, path_out, tar_files,no_compression=false)
+        #The cd is meant for cases when running cnvrg data uplaod not in the main folder
+        begin
+          if no_compression
+            `cd #{path_in} && tar -cf #{path_out} -T #{tar_files}`
+          else
+            `cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
+          end
+        rescue => e
+          puts "Exception while compressing data: #{e.message}"
+        end
+
+        return $?.success?
+      end
+
+      def extarct_tar(file_path, dir_path)
+        `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
+        return $?.success?
+      end
+      def cpu_usage
+        cpu_usage = 0.0
+        begin
+          cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
+        if !cpu.nil?
+          cpu_usage = cpu.to_f
+        end
+        rescue
+          cpu_usage = 0.0
+        end
+
+        return cpu_usage
+      end
+      def memory_usage
+        prec = 0.0
+        begin
+        used = `free -mt |grep Mem: |awk '{print $3}'`
+        total = `free -mt |grep Mem: |awk '{print $2}'`
+
+        used_f = used.to_f if !used.nil?
+        total_f = total.to_f if !total.nil?
+          prec = (used_f / total_f)*100
+          prec = prec.round(2)
+        rescue
+          prec = 0.0
+        end
+        return prec
+
+
+      end
+      def gpu_util
+        gpu = [0.0, 0.0]
+        begin
+        gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
+
+        if !gpu_stats.nil?
+          gpu = gpu_stats.strip
+          gpu = gpu_stats.gsub!("%","").split(",")
+          gpu[0] = gpu[0].to_f
+          gpu[1] = gpu[1].to_f
+          return gpu
+        end
+
+        rescue
+          return gpu
+        end
+
+
+      end
+      def usage_metrics_in_docker(docker_id)
+        res = [0.0,0.0]
+        begin
+          if docker_id.nil?
+            docker_id = `cat /etc/hostname`
+          end
+          stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
+          if !stats.nil?
+            conv = stats.split(",")
+            cpu = conv[0].gsub!("%","").to_f
+            memory = conv[1].gsub!("%","").to_f
+            res = [cpu,memory]
+            return res
+          end
+        rescue
+          return res
+        end
+
+
+
+      end
+
+      def with_progress_bar
+        not_done = true
+        upload_progress_bar = ProgressBar.create(:title => "Upload progress",
+                                                 :format => '%a <%B> %p%% %t',
+                                                 :starting_at => 0,
+                                                 :total => 600)
+        pb = Thread.new do
+          while not_done do
+            upload_progress_bar.increment
+            sleep(1)
+
+          end
+        end
 
+        yield.tap do # After yielding to the block, save the return value
+          not_done = false # Tell the thread to exit, cleaning up after itself…
+          pb.join # …and wait for it to do so.
+        end
+      end
     end
+
+
   end
 end