cnvrg 0.0.148 → 0.0.149

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9889a94a7cbc7cf8e729daf363e61bd38072960a
-  data.tar.gz: e46f0fed0b9b6a3007daa399dc4e75b81d402aa6
+  metadata.gz: 89c404e15ae2c15390c06632e16fdb46725a7dab
+  data.tar.gz: 28435401db7d5d2a78ba88de8946513ffc86dca5
 SHA512:
-  metadata.gz: 8bc2c1dca91ad6f2ff9e4248b95b34231bef97be4eb9d7ba52b5cd73c9def6ee36ad672beaf45c8581e868d0b89fc7d8cf6e6f6babfd35ed6f5feb5bf610c222
-  data.tar.gz: 02d1c07bec47cada6285f3df813b6fc67b0c3f2700df70bc89be691e72f20593da9672ce2fa03430dc2d548e5ba9ac930a7f3d2711b2a36e6fe32f022367e589
+  metadata.gz: 5442ff1a286b258d5624ba1bc00d7944423a99b0b74128cd1b90dffd24511e52e9609a7119ba2d24c4bec348927f7a637fb669b7f961d780f6070103e8cb8e20
+  data.tar.gz: 678610c32a2104cab60cec8bcab410cc5d76372079b99044776e444cc7713f206cb10b87521425459e3a8c53937032665a795fd36adfa48a58c9227a1963b998

data/lib/cnvrg/cli.rb

@@ -33,11 +33,13 @@ require 'active_support/all'
 require 'thor'
 require 'pathname'
 require 'cnvrg/data'
-
 # DEV VERSION
 
-#
+# include Cnvrg::Tar
+
+# include Tar
 module Cnvrg
+
   class CLI < Thor
 
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
@@ -50,10 +52,11 @@ module Cnvrg
         super
       end
     end
-    desc "data", "upload and manage datasets", :hide =>true
+    desc "data", "upload and manage datasets", :hide => true
     subcommand "data", Data
 
-    desc "", "" , :hide => true
+    desc "", "", :hide => true
+
     def test
       # image_settings = {
       #     'Image' => "cnvrg:latest",
@@ -74,11 +77,11 @@ module Cnvrg
       # s = "/leah/1/2/3/4/5"
       # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
       # puts container.exec(command, tty: true)
-      puts File.read("s4cmd.py")
     end
 
 
     desc 'version', 'Prints cnvrg current version'
+
     def version
       puts Cnvrg::VERSION
 
@@ -87,6 +90,7 @@ module Cnvrg
     map %w(-v --version) => :version
 
     desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
+
     def set_api_url(url)
       home_dir = File.expand_path('~')
       if !url.end_with? "/api"
@@ -153,7 +157,7 @@ module Cnvrg
     map %w(-api --api) => :set_api_url
 
     desc 'set_default_owner', 'set default owner'
-    
+
     def set_default_owner
 
       path = File.expand_path('~')+"/.cnvrg/config.yml"
@@ -177,7 +181,7 @@ module Cnvrg
           owners << choose_owner
           chosen = false
           while !chosen
-            owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
+            owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
             choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
 
             if choose_owner =~ /[[:digit:]]/
@@ -250,7 +254,7 @@ module Cnvrg
             owners << choose_owner
             chosen = false
             while !chosen
-              owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
+              owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
               choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
 
               if choose_owner =~ /[[:digit:]]/
@@ -417,7 +421,7 @@ module Cnvrg
         say "created project successfully", Thor::Shell::Color::GREEN
         say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurred, aborting", Thor::Shell::Color::RED
         if Dir.exist? working_dir
 
@@ -436,6 +440,7 @@ module Cnvrg
         exit(1)
       end
     end
+
     desc 'set_image', 'set_image for a project'
 
     def set_image(docker_image)
@@ -472,6 +477,7 @@ module Cnvrg
       @image.update_image_activity(nil, nil)
 
     end
+
     desc 'link', 'Link current directory to a new cnvrg project'
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
@@ -498,7 +504,7 @@ module Cnvrg
         end
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Project.link(owner,project_name, docker)
+        if Project.link(owner, project_name, docker)
           path = Dir.pwd
           @project = Project.new(path)
           @project.generate_idx()
@@ -556,10 +562,10 @@ module Cnvrg
       end
     end
 
-    desc 'init_data', 'Init dataset directory', :hide =>true
+    desc 'data init', 'Init dataset directory'
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
-    def init_data
+    def init_data(public)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -574,9 +580,10 @@ module Cnvrg
 
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Dataset.init(owner,dataset_name, options["public"])
+        if Dataset.init(owner, dataset_name, options["public"])
           path = Dir.pwd
           @dataset = Dataset.new(path)
+
           @dataset.generate_idx()
 
           url = @dataset.url
@@ -598,7 +605,117 @@ module Cnvrg
         exit(1)
       end
     end
-    desc 'data_snap', 'Init dataset directory', :hide =>true
+
+    desc 'data clone', 'Clone dataset'
+    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
+
+    def clone_data(dataset_url)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        url_parts = dataset_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
+        slug = url_parts[project_index+1]
+        owner = url_parts[project_index-1]
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
+
+        Cnvrg::CLI.is_response_success(response)
+        dataset_name = response["result"]["name"]
+        if (Dir.exists? dataset_name)
+          say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
+          if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+            say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
+            log_end(1, "conflict with dir #{dataset_name}")
+
+            exit(1)
+          end
+
+        end
+        if Dataset.clone(owner, dataset_name, slug)
+          say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
+
+          commit_to_clone = options["commit"] || nil
+          working_dir = "#{Dir.pwd}/#{dataset_name}"
+          @dataset = Dataset.new(working_dir)
+          @dataset.generate_idx()
+          say "Downloading data", Thor::Shell::Color::BLUE
+
+
+          download_data(false, false, path = working_dir)
+
+
+          check = Helpers.checkmark
+          say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
+          log_end(0)
+
+        else
+          log_end(1, "can't create dataset")
+          @dataset.revert(working_dir) unless @dataset.nil?
+          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          exit(0)
+        end
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'init_data_container', 'Init dataset directory', :hide => true
+    method_option :container, :type => :string, :aliases => ["--c"], :default => ""
+    method_option :login_content, :type => :string, :aliases => ["--l"], :default => ""
+    method_option :owner, :type => :string, :aliases => ["--o"], :default => ""
+    method_option :dataset_slug, :type => :string, :aliases => ["--ds"], :default => ""
+    method_option :dataset_name, :type => :string, :aliases => ["--dn"], :default => ""
+    method_option :sha1, :type => :string, :aliases => ["--s"], :default => ""
+    method_option :username, :type => :string, :aliases => ["--u"], :default => ""
+    method_option :api_url, :type => :string, :aliases => ["--a"], :default => ""
+
+    def init_data_container()
+      begin
+        container = options["container"]
+        login_content = options["login_content"]
+        dataset_slug = options["dataset_slug"]
+        owner = options["owner"]
+        dataset_name = options["dataset_name"]
+        sha1 = options["sha1"]
+        username = options["username"]
+        api_url = options["api_url"]
+
+        container = Docker::Container.get(container)
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "cnvrg set_remote_api_url #{owner} #{username} #{api_url}"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir -p /home/ds/notebooks/data/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrg/config.yml"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrgignore"]
+        container.exec(command, tty: true)
+
+        command = ["/bin/bash", "-lc", "cnvrg data_init_container #{owner} #{dataset_slug} #{dataset_name}"]
+        container.exec(command, tty: true)
+
+
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'data_snap', 'Init dataset directory', :hide => true
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
     def snap_data
@@ -607,11 +724,10 @@ module Cnvrg
         log_start(__method__, args, options)
 
         owner = CLI.get_owner
-          path = Dir.pwd
-          @dataset = Dataset.new(path)
-         puts @dataset.snapshot
+        path = Dir.pwd
+        @dataset = Dataset.new(path)
 
-          log_end(0)
+        log_end(0)
 
       rescue SignalException
         log_end(-1)
@@ -620,14 +736,116 @@ module Cnvrg
         exit(1)
       end
     end
+    desc 'data_snap', 'Init dataset directory', :hide => true
+
+    def data_init_container(owner,dataset_slug,dataset_name)
+      puts owner
+      puts dataset_name
+      puts dataset_slug
+
+      if Dataset.init_container(owner, dataset_slug, dataset_name)
+
+        say "init finished successfully", Thor::Shell::Color::GREEN
+        log_end(0)
+
+      else
+        log_end(1, "can't create dataset")
+        say "error creating dataset, please contact support.", Thor::Shell::Color::RED
+        exit(0)
+      end
+    end
+
+    desc 'data download', 'pull data'
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def download_data(verbose, sync, path=Dir.pwd)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        if path.nil? or path.empty?
+          path = Dir.pwd
+        end
+        dataset_dir = is_cnvrg_dir(path)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        new_branch = options["new_branch"] || false
+
+        res = @dataset.compare_idx(new_branch)["result"]
+
+        result = res["tree"]
+
+        commit = res["commit"]
+        if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
+          log_end(0)
+          return true
+        end
+        result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
+
+        delete = result["result"]["delete"]
+        commits = result["result"]["commits"]
+        updated_idx = result["result"]["idx"]
+        commits.each do |c|
+          file_name = @files.download_data_file(c, dataset_dir)
+
+          if file_name.eql? false or file_name.nil?
+            ##RETRY
+            next
+          end
+          file_path = "#{dataset_dir}/#{file_name}"
+          success = extarct_tar(file_path, dataset_dir)
+          if !success
+            #TODO: ?
+          end
+
+          FileUtils.rm_rf([file_path])
+
+        end
+        to_delete = []
+        delete.each do |d|
+          to_delete << "#{dataset_dir}/#{d}"
+        end
+        FileUtils.rm_rf(to_delete)
+
+        @dataset.update_idx(updated_idx)
+
+
+        # result["conflicts"].each do |f|
+        #   relative_path = f.gsub(/^#{@dataset.local_path}/, "")
+        #   if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
+        #     successful_changes << relative_path
+        #   end
+        #
+        # end
+        # update idx with latest commit
+        # @dataset.update_idx_with_commit!(commit)
+        check = Helpers.checkmark()
+        say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
+
+
+        log_end(0)
+      end
+    rescue
+      log_end(-1)
+
+      say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+      exit(1)
+    rescue SignalException
+      log_end(-1)
+      say "\nAborting", Thor::Shell::Color::BLUE
+      exit(1)
+    end
+
 
-    desc 'upload_data', 'Upload data files', :hide=>true
+    desc 'upload_data', 'Upload data files', :hide => true
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
     method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def upload_data(sync=false,direct=false)
+    def upload_data(sync=false, direct=false)
 
       begin
         verify_logged_in(true)
@@ -643,12 +861,12 @@ module Cnvrg
         result = @dataset.compare_idx(false)
 
         commit = result["result"]["commit"]
-          if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-            log_end(0)
+        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+          log_end(0)
 
-            say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
-            exit(1)
-          end
+          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+          exit(1)
+        end
 
         say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
         result = result["result"]["tree"]
@@ -745,7 +963,7 @@ module Cnvrg
           exit(0)
         end
         if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count +=  result["deleted"].size
+          update_count += result["deleted"].size
         end
         if update_count == update_total
           res = @files.end_commit(commit_sha1)
@@ -798,8 +1016,152 @@ module Cnvrg
 
           @files.rollback_commit(commit_sha1)
         end
-      rescue =>e
+      rescue => e
+
+        log_end(-1)
+
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      end
+
+    end
+
+    desc 'data upload', 'push data'
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+
+    def upload_data_tar(ignore, verbose, sync)
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        if !@dataset.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
+        result = @dataset.compare_idx(false)
+
+
+        commit = result["result"]["commit"]
+        if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+          log_end(0)
+
+          say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+          exit(1)
+        end
+
+        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
+        result = result["result"]["tree"]
+        check = Helpers.checkmark()
+
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_end(0)
+          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+        if verbose
+          if update_total == 1
+            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
+          else
+            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
+          end
+        else
+          say "Syncing files", Thor::Shell::Color::BLUE unless sync
+
+        end
+
+        # Start commit
+        res = @files.start_commit(false)["result"]
+        commit_sha1 =res["commit_sha1"]
+        commit_time = res["commit_time"]
+        # upload / update
+        begin
+          (result["added"] + result["updated_on_local"]).each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            successful_updates<< relative_path
+            update_count += 1
+          end
+
+          # delete
+          deleted = update_deleted(result["deleted"])
+          deleted.each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            successful_updates<< relative_path
+          end
+          @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
+
+          @dataset.update_idx_with_commit!(commit_sha1)
+
+          home_dir = File.expand_path('~')
+          tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
+          tar_files_path =  "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
+          tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
+          File.open(tar_files_path, 'w') {|f| f.write tar_files}
+          create_tar(dataset_dir, tar_path, tar_files_path)
+          res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
+          cur_idx = @dataset.get_idx.to_h
+          res = @files.end_commit_tar(commit_sha1, cur_idx)
+
+          # delete
+          FileUtils.rm_rf([tar_path,tar_files_path])
+
+        rescue SignalException
+          log_end(-1)
+          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+
+          @files.rollback_commit(commit_sha1)
+          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
+          FileUtils.rm_rf([tar_path]) if File.exist? tar_path
+
+          log_end(-1, e.message)
+          @files.rollback_commit(commit_sha1)
+          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          exit(0)
+        end
+        if verbose
+          say "#{check} Done", Thor::Shell::Color::BLUE
+          if successful_updates.size >0
+            say "Updated:", Thor::Shell::Color::GREEN
+            suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+            say suc.join("\n"), Thor::Shell::Color::GREEN
+          end
+          if successful_deletions.size >0
+            say "Deleted:", Thor::Shell::Color::GREEN
+            del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+            say del.join("\n"), Thor::Shell::Color::GREEN
+          end
+
+
+          say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+        else
+          say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
+        end
+
 
+      rescue => e
         log_end(-1)
 
         say "Error occurd, \nAborting", Thor::Shell::Color::RED
@@ -814,10 +1176,13 @@ module Cnvrg
         exit(1)
       end
 
+      log_end(0)
+
     end
 
 
-    desc 'unlink','Unlink a project from current directory', :hide=>true
+    desc 'unlink', 'Unlink a project from current directory', :hide => true
+
     def create_volume
       verify_logged_in(false)
       log_start(__method__, args, options)
@@ -827,8 +1192,9 @@ module Cnvrg
 
     end
 
-    desc 'list data','Unlink a project from current directory', :hide=>true
-    def list_data
+    desc 'list data', 'List all dataset you currently have'
+
+    def list_dataset
       verify_logged_in(true)
       log_start(__method__, args, options)
       dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -840,13 +1206,29 @@ module Cnvrg
 
     end
 
+    desc 'list data commits', 'List all commits for a specific dataset'
+
+    def list_dataset_commits()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      result = @dataset.list_commits()
+      list = result["result"]["list"]
+
+      print_table(list)
+
+    end
+
+
+    desc 'unlink', 'Unlink a project from current directory'
 
-    desc 'unlink','Unlink a project from current directory'
     def unlink
       verify_logged_in(false)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir()
-      list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
+      list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
       FileUtils.rm_rf list_to_del
     end
 
@@ -1024,7 +1406,9 @@ module Cnvrg
         exit(1)
       end
     end
-    desc '', '' , :hide => true
+
+    desc '', '', :hide => true
+
     def revert_exp
       begin
         log_start(__method__, args, options)
@@ -1033,7 +1417,7 @@ module Cnvrg
         result = @project.compare_idx(false)["result"]
         result = result["tree"]
         if result["added"].size > 0
-            FileUtils.rm_rf(result["added"])
+          FileUtils.rm_rf(result["added"])
         end
 
       rescue SignalException
@@ -1045,12 +1429,12 @@ module Cnvrg
 
 
     desc 'upload', 'Upload updated files'
-    method_option :ignore, :type => :array, :aliases => ["-i"], :desc => "ignore following files"
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def upload(link=false, sync=false,direct=false)
+    def upload(link=false, sync=false, direct=false, ignore_list="")
 
       begin
         verify_logged_in(true)
@@ -1059,7 +1443,19 @@ module Cnvrg
         @project = Project.new(get_project_home)
 
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
-        ignore = options[:ignore] || []
+        ignore = options[:ignore] || ""
+
+        if ignore.nil? or ignore.empty?
+          ignore = ignore_list
+        end
+        data_ignore = data_dir_include()
+        if !data_ignore.nil?
+          if ignore.nil? or ignore.empty?
+            ignore = data_ignore
+          else
+            ignore ="#{ignore},#{data_ignore}"
+          end
+        end
         if !@project.update_ignore_list(ignore)
           say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
         end
@@ -1072,7 +1468,6 @@ module Cnvrg
           end
         end
         result = @project.compare_idx(new_branch)
-
         commit = result["result"]["commit"]
         if !link
           if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
@@ -1177,7 +1572,7 @@ module Cnvrg
           exit(0)
         end
         if !result["deleted"].nil? and !result["deleted"].empty?
-          update_count +=  result["deleted"].size
+          update_count += result["deleted"].size
         end
         if update_count == update_total
           res = @files.end_commit(commit_sha1)
@@ -1235,8 +1630,7 @@ module Cnvrg
 
           @files.rollback_commit(commit_sha1)
         end
-      rescue =>e
-
+      rescue => e
         log_end(-1)
 
         say "Error occurd, \nAborting", Thor::Shell::Color::RED
@@ -1257,14 +1651,30 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
 
-    def download(sync=false)
+    def download(sync=false, ignore_list="")
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
         project_home = get_project_home
         @project = Project.new(project_home)
         @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        ignore = options[:ignore] || ""
+        if ignore.nil? or ignore.empty?
+          ignore = ignore_list
+        end
+        data_ignore = data_dir_include()
+        if !data_ignore.nil?
+          if ignore.nil? or ignore.empty?
+            ignore = data_ignore
+          else
+            ignore ="#{ignore},#{data_ignore}"
+          end
+        end
+        if !@project.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
         new_branch = options["new_branch"] || false
 
         res = @project.compare_idx(new_branch)["result"]
@@ -1347,7 +1757,7 @@ module Cnvrg
 
           log_end(0)
         end
-      rescue
+      rescue =>e
         log_end(-1)
 
         say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
@@ -1453,21 +1863,74 @@ module Cnvrg
     end
 
 
+    desc 'data_jump', 'jump to specific commit', :hide => true
+
+    def data_jump(*commit_sha1)
+      begin
+        verify_logged_in()
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        if commit_sha1.nil? or commit_sha1.empty?
+          commit_sha1 = @dataset.last_local_commit
+        end
+        response = @dataset.compare_commits(commit_sha1)
+        successful_changes = []
+        if !response["result"]["status"].nil?
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          status = response["result"]["status"]
+          (status["delete"]).each do |f|
+            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
+            FileUtils.rm_rf(relative_path)
+          end
+          # current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
+          (status["dirs"]).each do |f|
+            relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
+            # dir
+            if @files.download_dir(dataset_dir, relative_path)
+              # current_tree.delete(relative_path[0, relative_path.size-1])
+              successful_changes << relative_path
+            end
+          end
+          (status["download"]).each do |f|
+            relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
+            # dir
+            if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
+              successful_changes << relative_path
+            end
+          end
+
+
+          say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
+          log_end(0)
+        end
+
+      rescue SignalException
+        log_end(-1)
+        exi(1)
+      end
+    end
+
     desc 'sync', 'Sync with remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
 
     def sync(direct=true)
       verify_logged_in(true) if direct
       if options["verbose"]
-      say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
       else
         say 'Syncing project', Thor::Shell::Color::BLUE
       end
 
 
-      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
-      invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
+      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
+      invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+             :ignore => options[:ignore]
 
 
     end
@@ -1476,8 +1939,8 @@ module Cnvrg
     desc 'run cmd', 'Runs an experiment'
     method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
     method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
@@ -1491,6 +1954,9 @@ module Cnvrg
     method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
 
     def run(*cmd)
       sync_before = options["sync_before"]
@@ -1504,31 +1970,41 @@ module Cnvrg
       schedule = options["schedule"]
       image = options["image"]
       grid = options["grid"]
+      data = options["data"]
+      data_commit = options["data_commit"]
+      ignore = options["ignore"]
+
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
-               :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
+               :log => log, :email_notification => email_notification, :upload_output => upload_output,
+               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
         return
       else
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
-               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image, :grid => grid
+               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
+               :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
         return
       end
     end
 
 
-    desc '', '' , :hide => true
+    desc '', '', :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t"], :default => ""
     method_option :log, :type => :boolean, :aliases => ["-l", "--log"], :default => false
     method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
-    method_option :upload_output, :type => :string, :aliases => [ "-uo"], :default => ""
+    method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
-    method_option :indocker, :type => :boolean,  :default => false
+    method_option :indocker, :type => :boolean, :default => false
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+
 
     def exec(*cmd)
       log = []
@@ -1537,7 +2013,7 @@ module Cnvrg
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
-      script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
+      script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
 
 
       sync_before = options["sync_before"]
@@ -1545,8 +2021,10 @@ module Cnvrg
       print_log = options["log"]
       title = options["title"]
       commit = options["commit"] || nil
-      image = options["image"]  || nil
-      indocker = options["indocker"]  || false
+      image = options["image"] || nil
+      indocker = options["indocker"] || false
+      ignore = options[:ignore] || ""
+
 
       email_notification = options["email_notification"]
       upload_output = options["upload_output"]
@@ -1561,7 +2039,8 @@ module Cnvrg
         else
           if sync_before
             # Sync before run
-            invoke :sync, [false], :new_branch => is_new_branch
+
+            invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
           end
         end
         #set image for the project
@@ -1569,27 +2048,27 @@ module Cnvrg
           invoke :set_image, [image]
         end
         if !indocker
-        image_proj = is_project_with_docker(working_dir)
+          image_proj = is_project_with_docker(working_dir)
 
 
-        if image_proj and image_proj.is_docker
-          container= image_proj.get_container
-          if !container
-            say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
-            exit(1)
-          end
+          if image_proj and image_proj.is_docker
+            container= image_proj.get_container
+            if !container
+              say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
+              exit(1)
+            end
 
 
-          exec_args = args.flatten.join(" ")
-          options_hash = Hash[options]
-          options_hash.except!("image", "indocker")
-          exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-          command_to_run = cmd.join(" ")
-          command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
-          puts container.exec(command, tty: true)
-          container.stop()
-          exit(0)
-        end
+            exec_args = args.flatten.join(" ")
+            options_hash = Hash[options]
+            options_hash.except!("image", "indocker")
+            exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+            command_to_run = cmd.join(" ")
+            command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
+            puts container.exec(command, tty: true)
+            container.stop()
+            exit(0)
+          end
         end
         start_commit = @project.last_local_commit
         cmd = cmd.join("\s")
@@ -1602,7 +2081,7 @@ module Cnvrg
         machine_name = Socket.gethostname
         begin
           machine_activity = @exp.get_machine_activity(working_dir)
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -1660,7 +2139,7 @@ module Cnvrg
                   end
 
                 rescue Errno::EIO => e
-                    # break
+                  # break
                 rescue Errno::ENOENT
                   log_end(1, "command #{cmd} isn't valid")
 
@@ -1712,8 +2191,8 @@ module Cnvrg
                   # Sync after run
 
 
-                  download(sync=true)
-                  upload(link=false, sync=true,direct=false)
+                  download(sync=true, ignore_list=ignore)
+                  upload(link=false, sync=true, direct=false, ignore_list=ignore)
 
                 end
                 end_commit = @project.last_local_commit
@@ -1790,7 +2269,7 @@ module Cnvrg
         machine_name = Socket.gethostname
         begin
 
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -1944,19 +2423,37 @@ module Cnvrg
     method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
     method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
     method_option :machine_type, :type => :string, :default => ""
-    method_option :schedule, :type => :string, :aliases => [ "-s"], :default => ""
-    method_option :commit, :type => :string, :aliases => [ "-c"], :default => nil
+    method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
+    method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
+    method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
+    method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
+
 
     def exec_remote(*cmd)
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
-      path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
+      path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
 
       begin
         grid = options["grid"] || nil
+        data = options["data"] || nil
+        data_commit = options["data_commit"] || nil
+        ignore = options[:ignore] || ""
+
+        if !data.nil? and !data.empty?
+          if ignore.nil? or ignore.empty?
+            ignore = "data"
+          else
+            ignore +=",data"
+          end
+        end
+        if ignore.nil? or ignore.empty?
+          ignore= ""
+        end
 
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
@@ -1986,9 +2483,9 @@ module Cnvrg
           upload_output_option = "--upload_output=#{upload_output}"
         end
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "ignore", "data", "data_commit")
         exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
-        command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
+        command = "#{exec_options}  --ignore=#{ignore} #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
         if !schedule.nil? and !schedule.empty?
 
@@ -1999,7 +2496,7 @@ module Cnvrg
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image,[choose_image]
+          invoke :set_image, [choose_image]
         end
         image = is_project_with_docker(working_dir)
         if !image or !image.is_docker
@@ -2021,15 +2518,13 @@ module Cnvrg
         end
 
 
-
         invoke :sync, [false], []
 
         say "Running remote experiment", Thor::Shell::Color::BLUE
         exp = Experiment.new(project.owner, project.slug)
 
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
         if Cnvrg::CLI.is_response_success(res)
-
           # if res["result"]["machine"] == -1
           #   say "There are no available machines", Thor::Shell::Color::BLUE
           #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
@@ -2061,7 +2556,7 @@ module Cnvrg
           #   end
           # else
           check = Helpers.checkmark()
-          str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}" 
+          str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
 
           if res["result"]["grid"]
             str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
@@ -2101,7 +2596,6 @@ module Cnvrg
         schedule = options["schedule"] || ""
 
 
-
         if !schedule.nil? and !schedule.empty?
 
           local_timestamp = get_schedule_date
@@ -2126,7 +2620,6 @@ module Cnvrg
         end
 
 
-
         invoke :sync, [false], []
 
         res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
@@ -2186,15 +2679,16 @@ module Cnvrg
     method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
-    method_option :small, :type => :boolean, :aliases => ["-sm","--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
     method_option :image, :type => :string, :aliases => ["--i"], :default => ""
 
     desc 'notebook', 'starts a notebook session remotely or locally'
+
     def notebook
       local = options["local"]
       notebook_dir = options["notebook_dir"]
@@ -2202,14 +2696,14 @@ module Cnvrg
       image = options["image"]
 
       if local
-        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel,:image =>image
+        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
         return
       else
         instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
+                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
         instance_type = get_instance_type(instances)
 
-        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image =>image
+        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image
         return
 
       end
@@ -2219,7 +2713,7 @@ module Cnvrg
 
     desc 'remote_notebook', 'run notebook server on remote server'
     method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
-    method_option :machine_type, :type => :string,  :default => ""
+    method_option :machine_type, :type => :string, :default => ""
     method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
     method_option :image, :type => :string, :aliases => ["-i"], :default => ""
 
@@ -2239,7 +2733,7 @@ module Cnvrg
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image,[choose_image]
+          invoke :set_image, [choose_image]
         end
         @image = is_project_with_docker(working_dir)
         if !@image or !@image.is_docker
@@ -2250,7 +2744,7 @@ module Cnvrg
           end
           # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
           # if default
-            @image = Images.new(working_dir, default_image_name)
+          @image = Images.new(working_dir, default_image_name)
           # else
           #   exit(0)
           # end
@@ -2259,7 +2753,6 @@ module Cnvrg
         invoke :sync, [false], []
 
 
-
         res = @image.remote_notebook(notebook_dir, instance_type, kernel)
         if Cnvrg::CLI.is_response_success(res)
           if res["result"]["machine"] == -1
@@ -2368,7 +2861,7 @@ module Cnvrg
         end
       rescue => e
         log_end(-1, e.message)
-        say "Error occurred, aborting",Thor::Shell::Color::RED
+        say "Error occurred, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
         end
@@ -2473,7 +2966,7 @@ module Cnvrg
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
-          invoke :set_image,[choose_image]
+          invoke :set_image, [choose_image]
         end
         image = is_project_with_docker(Dir.pwd)
         if image and image.is_docker and !remote
@@ -2510,7 +3003,7 @@ module Cnvrg
         base_url = get_base_url()
 
         local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
-        command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+        command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
         container.exec(command, tty: true)
         container.stop()
         container.start()
@@ -2526,7 +3019,7 @@ module Cnvrg
           exit(1)
         end
 
-        url =  list[list.size-1].split(" ::")
+        url = list[list.size-1].split(" ::")
         token = url[0][(url[0].index("=")+1)..-1]
 
         # machine_activity = @note.get_machine_activity(project_dir)
@@ -2562,7 +3055,7 @@ module Cnvrg
           #   exit(1)
           # end
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
@@ -2583,6 +3076,7 @@ module Cnvrg
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+
     def notebook_stop
       begin
         verify_logged_in(true)
@@ -2638,7 +3132,7 @@ module Cnvrg
           exit(0)
         end
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -2688,7 +3182,7 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -2751,7 +3245,7 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurd, aborting"
         if container
           container.stop()
@@ -2766,6 +3260,7 @@ module Cnvrg
       end
 
     end
+
     desc 'build', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
@@ -2777,10 +3272,10 @@ module Cnvrg
         working_dir = is_cnvrg_dir
         install_file = options["install"] || nil
         if !install_file.nil?
-          commands = File.open(install_file).read.chop.gsub!("\n",",").split(",")
+          commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
 
         else
-          commands  = [cmd.join(" ")]
+          commands = [cmd.join(" ")]
         end
 
 
@@ -2798,25 +3293,25 @@ module Cnvrg
 
         end
         commands.each do |c|
-        if c.include? "pip"
-          c.sub("pip","/opt/ds/bin/pip")
-        end
-        if c.include? "pip3"
-          c.sub("pip3","/opt/ds3/bin/pip3")
-        end
+          if c.include? "pip"
+            c.sub("pip", "/opt/ds/bin/pip")
+          end
+          if c.include? "pip3"
+            c.sub("pip3", "/opt/ds3/bin/pip3")
+          end
 
-        say "Running #{c}", Thor::Shell::Color::BLUE
-        command = ["/bin/bash", "-lc", "#{c}"]
-        res = container.exec(command, tty: false)
-        if res[2] != 0
-         say  "Could not run command: #{c}, #{res[1][0]}" , Thor::Shell::Color::RED
-          container.stop()
-          log_end(0)
+          say "Running #{c}", Thor::Shell::Color::BLUE
+          command = ["/bin/bash", "-lc", "#{c}"]
+          res = container.exec(command, tty: false)
+          if res[2] != 0
+            say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
+            container.stop()
+            log_end(0)
 
-          exit(1)
-        end
-        say res[0].join("\n")
-        image.store_image_build_commands(working_dir,c)
+            exit(1)
+          end
+          say res[0].join("\n")
+          image.store_image_build_commands(working_dir, c)
         end
 
         checks = Helpers.checkmark()
@@ -2827,7 +3322,7 @@ module Cnvrg
 
         log_end(0)
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Error occurd, aborting", Thor::Shell::Color::RED
         if container
           container.stop()
@@ -2876,7 +3371,7 @@ module Cnvrg
         log_end(0)
         return new_image.id
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "\nError occurred,aborting"
         exit(1)
       rescue SignalException
@@ -2939,22 +3434,22 @@ module Cnvrg
         end
         if !name.nil? and !name.empty?
           if name.include? " "
-            name.gsub!(" ","_")
+            name.gsub!(" ", "_")
           end
         end
-        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n",",")
+        stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
         if stored_commands.nil? or stored_commands.empty?
           say "Nothing to push", Thor::Shell::Color::BLUE
           exit(0)
         end
 
         say "Pushing new image", Thor::Shell::Color::BLUE
-        if image.create_custom_image(name,working_dir,stored_commands)
+        if image.create_custom_image(name, working_dir, stored_commands)
 
           say "Image was updated successfully", Thor::Shell::Color::GREEN
         end
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "error occurred, aborting"
 
       end
@@ -3018,7 +3513,7 @@ module Cnvrg
           exit(1)
         end
       rescue => e
-        log_end(-1,e.message)
+        log_end(-1, e.message)
         say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
       rescue SignalException
         log_end(-1)
@@ -3044,9 +3539,8 @@ module Cnvrg
       container.start()
       cnvrg_command = cmd.join(" ")
       command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
-      res = container.exec(command, tty: true,wait:5400)[0]
+      res = container.exec(command, tty: true, wait: 5400)[0]
       say res
-
     end
 
     desc '', '', :hide => true
@@ -3054,8 +3548,13 @@ module Cnvrg
     def port_container(container_id)
       container = Docker::Container.get(container_id)
       say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
+    end
 
+    desc '', '', :hide => true
 
+    def tensor_port_container(container_id)
+      container = Docker::Container.get(container_id)
+      say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
     end
 
     desc '', '', :hide => true
@@ -3071,10 +3570,9 @@ module Cnvrg
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
     method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
-    method_option :volume, :type => :string, :aliases => ["-v", "--v"], :default => "/home/ds/notebooks/data"
 
 
-    def config_remote(image_name, port=7654,tensport=6006)
+    def config_remote(image_name, port=7654, tensport=6006)
       local_images = Docker::Image.all
 
       docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -3097,7 +3595,6 @@ module Cnvrg
             'ExposedPorts' => {
                 '8888/tcp' => {},
             },
-            'VolumesFrom' =>volume_from,
             'HostConfig' => {
                 'PortBindings' => {
                     '8888/tcp' => [
@@ -3124,7 +3621,7 @@ module Cnvrg
         say "#{container.id}:#{port}##{tensport}"
       rescue => e
         if e.message.include? "is not running"
-          return config_remote(image_name, port-1,tensport-1)
+          return config_remote(image_name, port-1, tensport-1)
         end
 
         if container
@@ -3133,13 +3630,14 @@ module Cnvrg
         return false
       end
     end
+
     desc '', '', :hide => true
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
     method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
 
 
-    def config_remote_gpu(image_name, port=7654,tensport=6006)
+    def config_remote_gpu(image_name, port=7654, tensport=6006)
       local_images = Docker::Image.all
 
       docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -3174,7 +3672,7 @@ module Cnvrg
         # }
 
         container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} #{image_name}:latest #{cmd} `
-        container_id = container_id.gsub("\n","")
+        container_id = container_id.gsub("\n", "")
         container = Docker::Container.get(container_id)
         # container.start()
         command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
@@ -3191,7 +3689,7 @@ module Cnvrg
       rescue => e
         if e.message.include? "is not running"
           puts "running asgain with: #{port-1} #{tensport-1}"
-          return config_remote_gpu(image_name, port-1,tensport-1)
+          return config_remote_gpu(image_name, port-1, tensport-1)
         end
 
         if container
@@ -3344,7 +3842,6 @@ module Cnvrg
             end
 
 
-
             return dir_path
 
           else
@@ -3378,13 +3875,13 @@ module Cnvrg
         owner = Cnvrg::CLI.get_owner()
         res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
         if Cnvrg::CLI.is_response_success(res)
-          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project",  "created by", "is_public", "last updated"
+          printf "%-20s %-20s  %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
           res["result"]["images"].each do |u|
             time = Time.parse(u["created_at"])
             update_at = get_local_time(time)
             created_by = u["created_by"]
 
-            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"],  created_by, u["is_public"], update_at
+            printf "%-20s %-20s  %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
           end
         end
         log_end(0)
@@ -3728,12 +4225,13 @@ module Cnvrg
           return owner
         end
       end
+
       def get_base_url
         home_dir = File.expand_path('~')
 
         config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
         api = config.to_h[:api]
-        return api.gsub!("/api","")
+        return api.gsub!("/api", "")
       end
 
       def get_project_home
@@ -3857,6 +4355,18 @@ module Cnvrg
           return false
         end
       end
+      def data_dir_include()
+        all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
+        all_dirs.flatten!
+        all_dirs.each do |a|
+          if a.include? "/.cnvrg"
+            ignore = File.dirname(a)
+            return ignore
+          end
+        end
+        return nil
+
+      end
 
       def verify_software_installed(software)
         begin
@@ -4032,20 +4542,21 @@ module Cnvrg
         return new_time
 
       end
+
       def update_deleted(deleted)
         final = []
         deleted.each do |d|
-            all_subs = d.split("/")
-            to_add = true
-            value = all_subs[0]
-            all_subs.each_with_index do |a,i|
-             if final.include? value+"/"
-               to_add = false
-               break
-             end
-              value = value+"/"+all_subs[i+1] if i <all_subs.size-1
+          all_subs = d.split("/")
+          to_add = true
+          value = all_subs[0]
+          all_subs.each_with_index do |a, i|
+            if final.include? value+"/"
+              to_add = false
+              break
             end
-            final << d if to_add
+            value = value+"/"+all_subs[i+1] if i <all_subs.size-1
+          end
+          final << d if to_add
 
 
         end
@@ -4053,7 +4564,8 @@ module Cnvrg
 
         return final
       end
-      def get_cmd_path_in_dir(main_dir,sub_dir)
+
+      def get_cmd_path_in_dir(main_dir, sub_dir)
         first = Pathname.new main_dir
         second = Pathname.new sub_dir
         relative = second.relative_path_from first
@@ -4064,10 +4576,18 @@ module Cnvrg
         end
       end
 
+      def create_tar(path_in, path_out, tar_files)
+        `tar -czf #{path_out} -T #{tar_files}  > /dev/null 2>&1`
+        return $?.success?
+      end
 
-
+      def extarct_tar(file_path, dir_path)
+        `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
+        return $?.success?
+      end
     end
   end
 end
 
 
+