cnvrg 0.5.0 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e44751ed52140fdaee81469a183a0f06f86968e1
-  data.tar.gz: 7b95fb6b3720a17d2edfe7a12da9c21ef0d3034f
+  metadata.gz: be48abaee7b44f00673ed8e4fe95fdc32044cdbf
+  data.tar.gz: fdc1bb3df689ea6524f268f910915559ba62ddfb
 SHA512:
-  metadata.gz: 2766aba6a508f5b073b4b12e987ee1d4568f84c47c632e6fefa6bf6fc0416a3737727cae88dfa7b97873c3f6461c5821d15a4ca9c7c54e32e9ca763b3e8b79d2
-  data.tar.gz: 8d2985f8c0ff5ea5d70131e993d896ce3ddd20a1f855dcf7ba7cad4a862a09fa13424bcdb3c5ebccb2a4e2ec9746f00eff2d7be49b7ee810826dc94d41a05a35
+  metadata.gz: 70a35b1ac372e78feffd115d7426338dced272bd351b7e76277f3dc2ead315d2632b9f28a07c4f7fd27f93c9668678addb4d91b106cb3c26b5e75d94832c752b
+  data.tar.gz: 96a10e4bd81552ea5b1ce22468fd102f9ec21a29e64152b83106a743e2e6278de0d68cb504fc78dcfb64098f30b60d5ecb051a224685dcdb5f9eea876ce12cc2

data/cnvrg.gemspec

@@ -26,7 +26,7 @@ Gem::Specification.new do |spec|
     spec.add_development_dependency 'pry'	
     
     spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
-    spec.add_runtime_dependency 'faraday', '~> 0.10.0'
+    spec.add_runtime_dependency 'faraday', '~> 0.15.2'
     spec.add_runtime_dependency 'netrc', '~> 0.11.0'
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
@@ -41,11 +41,10 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'launchy', '~> 2.4'
     spec.add_runtime_dependency 'docker-api', '~> 1.33'
     spec.add_runtime_dependency 'rubyzip', '~> 1.2'
-    spec.add_runtime_dependency 'activesupport', '~> 5.0'
+    spec.add_runtime_dependency 'activesupport', '~> 5.2.0'
     spec.add_runtime_dependency 'ruby-progressbar'
     spec.add_runtime_dependency 'net-ssh'
     spec.add_runtime_dependency 'down'
-    spec.add_runtime_dependency 'faraday_middleware-request-retry'
 
 
 

data/lib/cnvrg/api.rb

@@ -81,8 +81,8 @@ module Cnvrg
                     response
                 end
                 when 'POST'
-                conn.options.timeout = 420
-                conn.options.open_timeout =420
+                conn.options.timeout = 4200
+                conn.options.open_timeout =4200
                 retries = 0
                 success = false
                 while !success and retries < 20
@@ -121,8 +121,8 @@ module Cnvrg
                     fr.request :retry, max: 2, interval: 0.05,interval_randomness: 0.5, backoff_factor: 2
                     fr.adapter :net_http
                 end
-                conn.options.timeout = 420
-                conn.options.open_timeout =420
+                conn.options.timeout = 4200
+                conn.options.open_timeout =4200
 
 
                 # what if windows?

data/lib/cnvrg/cli.rb

@@ -147,7 +147,7 @@ module Cnvrg
     IP = "localhost"
     PORT = 7654
 
-    ParallelThreads ||= 10
+    ParallelThreads ||= 15
     ParallelProcesses ||= Parallel.processor_count
 
     class << self
@@ -647,6 +647,7 @@ module Cnvrg
     desc 'link', 'Link current directory to a new cnvrg project'
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
+    method_option :git, :type => :boolean, :aliases => ["-g","--git"], :default => false
 
     def link
       begin
@@ -660,6 +661,8 @@ module Cnvrg
         end
 
         sync = options["sync"]
+        git = options["git"] ||  false
+
         project_name = File.basename(Dir.getwd)
         log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
         if File.directory?(Dir.getwd + "/.cnvrg")
@@ -669,7 +672,7 @@ module Cnvrg
         end
         working_dir = Dir.getwd
         owner = CLI.get_owner
-        if Project.link(owner, project_name, docker)
+        if Project.link(owner, project_name, docker = docker,git = git)
           path = Dir.pwd
           @project = Project.new(path)
           @project.generate_idx()
@@ -793,7 +796,9 @@ module Cnvrg
     desc 'data clone', 'Clone dataset'
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
-    def clone_data(dataset_url,only_tree=false,commit=nil)
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+
+    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -805,7 +810,8 @@ module Cnvrg
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
-        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit})
+
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'POST',{ commit: commit, query:query})
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["result"]["name"]
         dataset_home = Dir.pwd+"/"+dataset_name
@@ -819,12 +825,21 @@ module Cnvrg
           end
 
         end
+        check = Helpers.checkmark
+
         if Dataset.clone(owner, dataset_name, slug)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
           @files = Cnvrg::Datafiles.new(owner, slug)
 
           successful_changes = []
           log_message("Downloading files", Thor::Shell::Color::BLUE)
+          if not(query.nil? or query.empty?)
+            @files.download_list_files_in_query(response["result"]["list_files"],dataset_home)
+
+            log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+            return
+
+          end
           commit = response["result"]["commit"]
           idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
           File.open(dataset_home + "/.cnvrg/idx.yml", "w+") {|f| f.write idx.to_yaml}
@@ -878,8 +893,6 @@ module Cnvrg
 
 
           end
-          check = Helpers.checkmark
-
           successful_changes = response["result"]["tree"]
           if !successful_changes.nil? and is_success
             Dataset.verify_cnvrgignore_exist(dataset_name, false)
@@ -899,10 +912,84 @@ module Cnvrg
         say "\nAborting"
         exit(1)
       end
+    end
 
+    desc 'data clone_query', 'Clone dataset _query'
+    method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => ""
+    def clone_data_query(dataset_url,query=nil)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        query = options["query"] || query
+        if !query.present?
+          log_message("Argument missing : query", Thor::Shell::Color::RED)
+          exit(1)
+        end
 
+        url_parts = dataset_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
+        slug = url_parts[project_index + 1]
+        owner = url_parts[project_index - 1]
 
+        response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/search/#{query}", 'GET')
+        Cnvrg::CLI.is_response_success(response,true)
+        dataset_name = response["results"]["name"]
+        dataset_slug = response["results"]["slug"]
+        # dataset_home = Dir.pwd+"/"+dataset_name
+        dataset_home = Dir.pwd
 
+        if Dataset.blank_clone(owner, dataset_name, dataset_slug)
+          dataset = Dataset.new(dataset_home)
+          log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
+          parallel_options = {
+              :progress => {
+                  :title => "Download Progress",
+                  :progress_mark => '=',
+                  :format => "%b>>%i| %p%% %t",
+                  :starting_at => 0,
+                  :total => response["results"]["query_files"].size,
+                  :autofinish => true
+              },
+              in_threads: ParallelThreads
+          }
+          begin
+            log_message("Downloading files", Thor::Shell::Color::BLUE)
+            Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
+              relative_path = f["fullpath"].gsub(/^#{dataset_home}/, "")
+              relative_path_dir = relative_path.split("/")
+              file_name = relative_path_dir.pop()
+              relative_path_dir = relative_path_dir.join("/")
+              abs_path = dataset_home + "/" + relative_path_dir
+              begin
+                FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
+              rescue
+                log_message("Could not create directory: #{abs_path}", Thor::Shell::Color::RED)
+                exit(1)
+              end
+              begin
+                File.write "#{abs_path}/#{file_name}", open(f["s3_url"]).read unless File.exist? (abs_path + "/" + file_name)
+              rescue
+                log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
+                exit(1)
+              end
+
+            end
+          rescue Interrupt
+            log_message("Couldn't download", Thor::Shell::Color::RED)
+            exit(1)
+          end
+          begin
+            dataset.generate_idx()
+            check = Helpers.checkmark
+            log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
+          rescue
+              exit(1)
+          end
+        end
+      rescue SignalException
+        say "\nAborting"
+        exit(1)
+      end
     end
 
     desc 'init_data_container', 'Init dataset directory', :hide => true
@@ -1210,7 +1297,11 @@ module Cnvrg
           if (Cnvrg::CLI.is_response_success(res, false))
             # save idx
             begin
-              @dataset.update_idx_with_files_commits!((successful_deletions + successful_updates), res["result"]["commit_time"])
+              list_files = []
+              list_files.concat successful_deletions
+              list_files.concat successful_updates
+
+              @dataset.update_idx_with_files_commits!(list_files, res["result"]["commit_time"])
 
               @dataset.update_idx_with_commit!(commit_sha1)
             rescue => e
@@ -1490,7 +1581,44 @@ module Cnvrg
         print_table(list)
 
 
+    end
+
+    desc 'data queries', 'List all data search queries you currently have'
+
+    def queries
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      result = @dataset.search_queries()
+      print_table(result)
+    end
+
+    desc 'data download_tags_yaml', 'Download dataset tags yml file in current directory'
+
+    def download_tags_yaml
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      status = @dataset.download_tags_yaml()
+      if status
+        log_message("Downloaded tags yaml successfully", Thor::Shell::Color::GREEN)
+      else
+        log_message("Unable to download", Thor::Shell::Color::RED)
       end
+    end
+
+    desc 'data query_files', 'List all data search queries you currently have'
+    def query_files(query)
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      query = options["query"] || query
+      result = @dataset.get_query_file(query)
+      print_table(result)
+    end
 
     desc 'data commits', 'List all commits for a specific dataset'
 
@@ -1532,6 +1660,51 @@ module Cnvrg
       FileUtils.rm_rf list_to_del
     end
 
+
+
+    desc 'git_clone', 'Clone project'
+    def git_clone(slug, owner)
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+
+      clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
+      idx_status = Project.new(get_project_home).generate_idx
+    end
+
+
+    desc 'link git project', 'link git project'
+    method_option :git, :type => :boolean, :aliases => ["-g", "--git"], :default => false
+    def link_git(project_url)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        url_parts = project_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
+        slug = url_parts[project_index + 1]
+        owner = url_parts[project_index - 1]
+        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
+        Cnvrg::CLI.is_response_success(response)
+        response = JSON.parse response["result"]
+        project_name = response["title"]
+
+        log_message("Linking #{project_name}", Thor::Shell::Color::BLUE)
+        clone_resp = Project.clone_dir_remote(slug, owner, project_name, true)
+        idx_status = Project.new(get_project_home).generate_idx
+        log_message("Linking project #{project_name} successfully", Thor::Shell::Color::GREEN)
+
+        return
+      rescue => e
+        log_message("Error occurred, \nAborting", Thor::Shell::Color::RED)
+        log_error(e)
+        return
+      rescue SignalException
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        return
+      end
+
+    end
+
     desc 'clone', 'Clone project'
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
@@ -1545,17 +1718,34 @@ module Cnvrg
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
         remote = options["remote"] || false
+
         response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
         Cnvrg::CLI.is_response_success(response)
         response = JSON.parse response["result"]
         project_name = response["title"]
+        git = response["git"] || false
+
         commit_to_clone = options["commit"] || nil
 
         log_message("Cloning #{project_name}", Thor::Shell::Color::BLUE)
         clone_resp = false
-        if remote
-          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
-          project_home = Dir.pwd
+        project_home = Dir.pwd
+
+        if remote and !git
+          clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
+        elsif git
+          if remote
+            clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
+          else
+            project_home += "/#{project_name}"
+            clone_resp = Project.clone_dir(slug, owner, project_name,git)
+
+          end
+
+          idx_status = Project.new(project_home).generate_idx
+          log_message("Cloned project #{project_name} successfully", Thor::Shell::Color::GREEN)
+
+          return
         else
           if (Dir.exists? project_name)
             # project_name = "#{project_name}_#{rand(1 .. 5000000000)}"
@@ -1567,7 +1757,7 @@ module Cnvrg
             end
 
           end
-          clone_resp = Project.clone_dir(slug, owner, project_name)
+          clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
 
 
@@ -1766,21 +1956,25 @@ module Cnvrg
         exit(1)
       end
     end
-    desc 'sync_data_new', 'sync_data_new'
+    desc 'sync_data_new', 'sync_data_new', :hide=> true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
+    method_option :tags, :type => :string, :aliases => ["--tags"], :desc => "upload file tags", :default => ""
     method_option :commit, :type => :string, :aliases => ["-c"], :desc => "download specified commit", :default => nil
     method_option :all_files, :type => :boolean, :aliases => ["--all"], :desc => "download specified commit", :default => true
+    method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
-    def sync_data_new(new_branch, force, verbose, commit, all_files)
+    def sync_data_new(new_branch, force, verbose, commit, all_files, tags,parallel)
       verify_logged_in(true)
       log_start(__method__, args, options)
       log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
       if !options[:force]
         invoke :download_data_new,[verbose,true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
-      invoke :upload_data_new,[new_branch,  verbose,true,force], :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true
+      invoke :upload_data_new,[new_branch,  verbose,true,force, tags], :new_branch=>new_branch,
+             :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
 
     end
     desc 'upload_data_new', 'upload_data_new'
@@ -1788,10 +1982,34 @@ module Cnvrg
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s","--sync"], :default => false
-    def upload_data_new(new_branch,  verbose,sync,force)
-      commit = invoke :start_commit_data,[], :new_branch=>new_branch, :direct=>false, :force =>force
+    method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
+    method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
+
+    def upload_data_new(new_branch,  verbose,sync,force, tags)
+      commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force
+      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose,
+                          :force =>force, :sync =>sync,:parallel => options[:parallel]
+      if upload_res
+        invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force
+      end
+      if tags
+        log_message('Uploading Tags', Thor::Shell::Color::BLUE)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+        begin
+          tag_file = File.open(options[:tags], "r+")
+          status = @dataset.upload_tags_via_yml(tag_file)
+        rescue
+          log_message('Tags file not found', Thor::Shell::Color::RED)
+          return
+        end
+        if status
+          log_message('Tags are successfully uploaded', Thor::Shell::Color::GREEN)
+        else
+          log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
+        end
+      end
 
-      upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync
       if upload_res
         invoke :end_commit_data,[commit] , :new_branch=>new_branch,  :force =>force
       end
@@ -1925,6 +2143,9 @@ module Cnvrg
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :tags, :type => :string, :aliases => ["--tags"], :desc => "upload file tags", :default => ""
+    # method_option :tags_yml, :type => :boolean, :aliases => ["--file_tag_yml"], :default => false
+    method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
 
     def upload_data_files(new_commit, *files)
 
@@ -1944,10 +2165,10 @@ module Cnvrg
         end
 
         force = options[:force] || false
-
+        parallel_threads = options["parallel"]  || ParallelThreads
         new_branch = options["new_branch"] || false
           log_message("Checking dataset", Thor::Shell::Color::BLUE) unless options[:sync]
-          local_idx = @dataset.generate_idx
+          local_idx = @dataset.generate_idx(true)
           result = @dataset.compare_idx(new_branch, commit=@dataset.last_local_commit, local_idx= local_idx, force=force, next_commit= next_commit)
 
         commit_sha1 = result["result"]["commit"]
@@ -1987,7 +2208,6 @@ module Cnvrg
 
         # upload / update
         begin
-
           parallel_options = {
               :progress => {
                   :title => "Upload Progress",
@@ -1997,7 +2217,7 @@ module Cnvrg
                   :total => (result["added"] + result["updated_on_local"]).size,
                   :autofinish => true
               },
-              in_threads: ParallelThreads,
+              in_threads: parallel_threads.to_i,
               isolation: true
           }
           successful_updates = []
@@ -2152,10 +2372,10 @@ module Cnvrg
     method_option :message, :type => :string, :aliases => ["-m", "--message"], :default => ""
     method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
-    method_option :files, :type => :string, :aliases => ["-f", "--files"], :default => nil
-
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false)
+    method_option :files, :type => :string, :aliases => ["--files"], :default => nil
+    method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
 
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false)
       begin
         verify_logged_in(true)
         log_start(__method__, args, options)
@@ -2173,6 +2393,11 @@ module Cnvrg
         if !spec_files_to_upload.blank?
           spec_files_to_upload = spec_files_to_upload.split(",")
         end
+        git_output_dir = options["output_dir"]
+        if !git_output_dir.blank?
+          spec_files_to_upload = Dir.glob("#{git_output_dir}/**/*", File::FNM_DOTMATCH).flatten
+          force = true
+        end
 
         if ignore.nil? or ignore.empty?
           ignore = ignore_list
@@ -2201,6 +2426,7 @@ module Cnvrg
           log_message("Comparing local changes with remote version:", Thor::Shell::Color::BLUE, (options["verbose"]))
         end
         result = result["result"]["tree"]
+
         check = Helpers.checkmark()
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
@@ -2685,8 +2911,42 @@ module Cnvrg
         say "\nAborting", Thor::Shell::Color::BLUE
         exit(1)
       end
+      end
+    desc 'download in git project', 'Download other files', :hide =>true
+    def download_in_git(commit_sha1)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        project_home = get_project_home
+        @project = Project.new(project_home)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        res = @project.git_download_commit(commit_sha1)
+        files =  res.try(:fetch, "result").try(:fetch, "files")
+        if files.blank?
+          log_message("No files to download", Thor::Shell::Color::RED)
+          return
+        end
+
+        begin
+          download_result = @files.download_multiple_files_s3(files, project_home)
+          if download_result
+            log_message("Done.\nDownloaded  finished successfully", Thor::Shell::Color::GREEN)
+          else
+            log_message("Error while trying to download\n", Thor::Shell::Color::RED)
+
+          end
+
+            rescue => e
+          log_error(e)
+          puts e.backtrace
+          log_message("Error while trying to download ", Thor::Shell::Color::RED)
+          return
+        end
+
+      end
     end
 
+
     desc 'download', 'Download updated files'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -2727,19 +2987,6 @@ module Cnvrg
           log_message("Project is up to date", Thor::Shell::Color::GREEN, ((options["sync"] or sync) ? false : true))
           return true
         end
-        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
-        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
-        #   if all.size == 1
-        #     num = "1 conflict"
-        #   else
-        #     num =  "#{result["conflicts"].size} conflicts"
-        #   end
-        #   say "Project contains #{num}:", Thor::Shell::Color::RED
-        #   say "#{all.join("\n")}"
-        #   say "Please fix them, and retry", Thor::Shell::Color::RED
-        #   exit(1)
-        #   end
-        update_count = 0
         update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
 
 
@@ -2906,7 +3153,6 @@ module Cnvrg
 
     desc 'jump', 'jump to specific commit'
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
-
     def jump(commit_sha1)
       begin
         verify_logged_in()
@@ -3073,18 +3319,21 @@ module Cnvrg
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :deploy, :type => :boolean, :aliases => ["-d", "--deploy"], :default => false
     method_option :in_exp, :type => :boolean, :aliases => ["-e", "--in_exp"], :default => false
-    method_option :files, :type => :string, :aliases => ["-f", "--files"], :default => nil
+    method_option :files, :type => :string, :aliases => ["--files"], :default => nil
+    method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+
 
     def sync(direct = true)
       verify_logged_in(true) if direct
       log_start(__method__, args, options)
       log_message('Checking for new updates from remote version', Thor::Shell::Color::BLUE, options["verbose"])
       log_message('Syncing project', Thor::Shell::Color::BLUE, !options["verbose"])
-      if !options[:force] and options[:files].blank?
+      if !options[:force] and (options[:files].blank? or options[:output_dir].blank?)
         invoke :download, [true, "", options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
       invoke :upload, [false, true,  direct, "",options["in_exp"] ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
-             :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"], :files => options["files"]
+             :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
+             :files => options["files"], :output_dir => options["output_dir"]
 
 
     end
@@ -3107,7 +3356,7 @@ module Cnvrg
     method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
+    method_option :image, :type => :string, :aliases => ["--image"], :default => ""
     method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
@@ -3117,6 +3366,11 @@ module Cnvrg
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => nil #//15,30,45,60
     method_option :max_time, :type => :string, :aliases => [ "--max_time"], :default => nil
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
+    method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
+    method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :git_commit, :type => :string, :aliases => [ "--git_commit"], :default => nil
+    method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
+
 
     def run(*cmd)
       verify_logged_in(true)
@@ -3141,12 +3395,26 @@ module Cnvrg
       max_time = options["max_time"]
       dataset_only_tree = options["dataset_only_tree"]
       custom_machine = options["machine"]
+      output_dir = options["output_dir"]
+      data_query = options["data_query"]
+      if !data.present? and data_query.present?
+        log_message("Please provide data with data_query", Thor::Shell::Color::RED)
+        exit(1)
+      end
+      if data_query.present? and (data_commit.present? or dataset_only_tree.present?)
+        log_message("Please use only one option: --query(-q) or #{data_commit.present? ? '--data_commit' : '--dataset_only_tree'} ", Thor::Shell::Color::RED)
+        exit(1)
+      end
+      git_commit = options["git_commit"]
+      git_branch = options["git_branch"]
+
 
       options_hash = Hash[options]
+
       if local
         invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
                :log => log, :email_notification => email_notification, :upload_output => upload_output,
-               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force
+               :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
         return
       else
         if !periodic_sync.nil? and !periodic_sync.empty?
@@ -3161,15 +3429,16 @@ module Cnvrg
 
           end
         end
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
+        instances = { "small" => options["small"], "medium" => options["medium"], "large" => options["large"],
                       "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"],
-                     options["machine"] => !options["machine"].blank?}
+                     options["machine"] => !options["machine"].blank? }
         instance_type = get_instance_type(instances)
         invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
                :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
                :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
                 :max_time => max_time,
-               :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree
+               :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
+               :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch
         return
       end
 
@@ -3193,6 +3462,9 @@ module Cnvrg
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
     method_option :sync_before_terminate, :type => :boolean, :aliases => ["-sbt", "--sync_before_terminate"], :default => false
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => ""
+    method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
+    method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+
     def exec(*cmd)
       log = []
       verify_logged_in(true)
@@ -3215,12 +3487,14 @@ module Cnvrg
 
 
       email_notification = options["email_notification"]
-      upload_output = options["upload_output"]
-      upload_output = "1m" if upload_output.nil? or upload_output.empty?
-      time_to_upload = calc_output_time(upload_output)
       project_home = get_project_home
+      data_query = options["data_query"]
       @project = Project.new(project_home)
-      is_new_branch = @project.compare_commit(commit)
+      if @project.is_git
+        sync_before = false
+      end
+      # is_new_branch = @project.compare_commit(commit)
+      is_new_branch = false
       begin
         if !commit.nil? and !commit.empty?
           invoke :jump, [commit], []
@@ -3287,13 +3561,30 @@ module Cnvrg
                   spot_status_thread = Thread.new do
                     begin
                       loop do
+                        puts "Checking Spot Instance Status"
+                        log_message('Checking Spot Instance Status', Thor::Shell::Color::YELLOW)
                         restart = @exp.restart_spot_instance()
+
                         if restart
                           log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
                           # sync
-                          download(sync = true, ignore_list = [])
-                          upload(link = false, sync = true, direct = false, ignore_list = [])
-                          @exp.send_restart_request()
+                          if @project.is_git
+                            output_dir = @exp.output_dir
+
+                            if output_dir.blank?
+                              output_dir = "output"
+                            end
+                            invoke :upload, [false, true, false, ignore, true, true], :output_dir => output_dir, :force=>true
+                          else
+                            upload_res = upload(false, true,  false,  ignore, true, true)
+
+                          end
+                            res = @exp.send_restart_request(@project.get_idx.try(:fetch, :commit))
+                          while !Cnvrg::CLI.is_response_success(res, false) do
+                            sleep(5)
+                            res = @exp.send_restart_request(@project.get_idx.try(:fetch, :commit))
+                          end
+                          exit(0)
                         end
                         sleep(10)
                       end
@@ -3321,23 +3612,7 @@ module Cnvrg
                   end
                 end
               end
-              log_thread = Thread.new do
-                while process_running or !log.empty? do
-                  begin
-                    temp_log = log
-                    if !temp_log.empty?
-                      @exp.upload_temp_log(temp_log) unless temp_log.empty?
-
-                    end
-                    log -= temp_log
-                  rescue => e
-                    log_message("Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW)
-                    log_error(e)
-                  ensure
-                    sleep 10
-                  end
-                end
-              end
+              start_time = Time.now
               PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
                 begin
                   stdout.each do |line|
@@ -3348,15 +3623,18 @@ module Cnvrg
                                type: "stdout",
                                real: real_time
                     }
-                    $LOG.info(cur_log)
                     if print_log
                       puts cur_log
                     end
                     log << cur_log
-                    # if log.size >= 10
-                    #   @exp.upload_temp_log(log)
-                    #   log = []
-                    # end
+                    if log.size >= 15
+                      @exp.upload_temp_log(log) unless log.empty?
+                      log = []
+                      elsif (start_time + 30.seconds) <= Time.now
+                      @exp.upload_temp_log(log) unless log.empty?
+                      log = []
+                      start_time = Time.now
+                    end
                   end
                   if stderr
                     stderr.each do |err|
@@ -3366,10 +3644,8 @@ module Cnvrg
                 rescue Errno::EIO => e
                   log_error(e)
                   if !log.empty?
-
                     temp_log = log
                     @exp.upload_temp_log(temp_log) unless temp_log.empty?
-
                     log -= temp_log
                   end
                 rescue Errno::ENOENT => e
@@ -3380,7 +3656,6 @@ module Cnvrg
                 #   exp_success = false
                 #   log_message("The process exited!", Thor::Shell::Color::RED)
                 rescue => e
-                  sleep(20) # end cycle
                   res = @exp.end(log, 1, start_commit, 0, 0)
                   log_message("Error occurred,aborting", Thor::Shell::Color::RED)
                   log_error(e)
@@ -3394,7 +3669,6 @@ module Cnvrg
 
                 temp_log = log
                   @exp.upload_temp_log(temp_log) unless temp_log.empty?
-
                 log -= temp_log
               end
 
@@ -3410,8 +3684,17 @@ module Cnvrg
                 upload_res_count = 0
                 upload_res = false
                   while !upload_res and upload_res_count <5
+                    if @project.is_git
+                      if !upload_output.blank?
+                        upload_output = upload_output.join(",")
+                        invoke :upload, [false, true, false, ignore, true, true], :output_dir => upload_output, :force=>true
+                      end
+
+                    else
+                      upload_res = upload(false, true,  false,  ignore, true, false)
+
 
-                    upload_res = upload(false, true,  false,  ignore, true)
+                    end
                     upload_res_count +=1
                   end
                 # download(sync = true, ignore_list = ignore)
@@ -3420,7 +3703,7 @@ module Cnvrg
                 end
                 end_commit = @project.last_local_commit
 
-                log_thread.join
+                # log_thread.join
                 stats_thread.join
 
               res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
@@ -3440,19 +3723,13 @@ module Cnvrg
               end
               log_message("Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED)
               if @exp
-                cur_time = Time.now
-                real_time = Time.now - real
-                cur_log = {time: cur_time,
-                           message: "Couldn't run #{cmd}, check your input parameters",
-                           type: "stdout",
-                           real: real_time
-
-                }
-                log << cur_log
-                process_running = false
-                log_thread.join
-                stats_thread.join
-                res = @exp.end(log, "-1", end_commit, cpu_average, memory_average)
+                # log_thread.join
+                Thread.kill(stats_thread)
+                exit_status = $?.exitstatus
+                if exit_status.blank?
+                  exit_status = "-1"
+                end
+                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
 
               end
               log_error(e)
@@ -3470,7 +3747,7 @@ module Cnvrg
         exit_status = -1
         end_commit = @project.last_local_commit
         process_running = false
-        log_thread.join
+        # log_thread.join
         stats_thread.join
 
         res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
@@ -3502,6 +3779,10 @@ module Cnvrg
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => nil
     method_option :sync_before_terminate, :type => :boolean, :aliases => ["-sbt", "--sync_before_terminate"], :default => false
+    method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
+    method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :git_commit, :type => :string, :aliases => [ "--git_commit"], :default => nil
+    method_option :git_branch, :type => :string, :aliases => [ "--git_branch"], :default => nil
 
     def exec_remote(*cmd)
       verify_logged_in(true)
@@ -3513,6 +3794,7 @@ module Cnvrg
         grid = options["grid"] || nil
         data = options["data"] || nil
         data_commit = options["data_commit"] || nil
+        data_query = options["data_query"] || nil
         sync_before = options["sync_before"]
         force = options["force"]
         max_time = options["max_time"]
@@ -3530,7 +3812,6 @@ module Cnvrg
         if dataset_only_tree
           ds_sync_options = 1
         end
-
         instance_type = options["machine_type"] || nil
         schedule = options["schedule"] || ""
         if schedule.start_with? 'in'
@@ -3562,9 +3843,14 @@ module Cnvrg
         if !instance_type.nil? and instance_type.include? "gpu"
           remote = "#{remote} --gpu=true"
         end
+        output_dir = options["output_dir"] || nil
+        git_commit = options["git_commit"]
+        git_branch = options["git_branch"]
 
         options_hash = Hash[options]
-        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree")
+        options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit",
+                             "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
+                             "data_query", "git_commit","git_branch" )
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -3574,7 +3860,9 @@ module Cnvrg
 
         end
         project = Project.new(working_dir)
-
+        if project.is_git and output_dir.blank?
+          output_dir  = "output"
+        end
         choose_image = options["image"]
 
         if !choose_image.nil? and !choose_image.empty?
@@ -3592,7 +3880,7 @@ module Cnvrg
           image_slug = image.image_slug
         end
         forced_commit = nil
-        if sync_before
+        if sync_before and !project.is_git
           if force
           sync_result = invoke :sync, [false], :force => force, :return_id=> true
           begin
@@ -3606,6 +3894,7 @@ module Cnvrg
 
 
 
+
           end
 
 
@@ -3629,7 +3918,7 @@ module Cnvrg
           commit_to_run = forced_commit
         end
         res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
-                              periodic_sync, sync_before_terminate, max_time, ds_sync_options)
+                              periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,data_query, git_commit, git_branch)
         if Cnvrg::CLI.is_response_success(res)
           check = Helpers.checkmark()
           str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
@@ -3750,11 +4039,13 @@ module Cnvrg
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
-
+    method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => ""
 
     desc 'notebook', 'starts a notebook session remotely or locally'
 
     def notebook
+      verify_logged_in(true)
+      log_start(__method__, args, options)
       local = options["local"]
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
@@ -3762,6 +4053,16 @@ module Cnvrg
       data = options["data"]
       data_commit = options["data_commit"]
       dataset_only_tree = options["dataset_only_tree"]
+      data_query = options["data_query"]
+      if !data.present? and data_query.present?
+        log_message("Please provide data with data_query", Thor::Shell::Color::RED)
+        exit(1)
+      end
+      if data_query.present? and (data_commit.present? or dataset_only_tree.present?)
+        log_message("Please use only one option: --query(-q) or #{data_commit.present? ? '--data_commit' : '--dataset_only_tree'} ", Thor::Shell::Color::RED)
+        exit(1)
+      end
+
       if local
         invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
         return
@@ -3771,7 +4072,7 @@ module Cnvrg
         instance_type = get_instance_type(instances)
 
         invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
-               :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree
+               :data => data, :data_commit => data_commit , :dataset_only_tree => dataset_only_tree, :data_query => data_query
         return
 
       end
@@ -3891,7 +4192,7 @@ module Cnvrg
     method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
     method_option :commit, :type => :string, :aliases => ["--commit"], :default => ""
     method_option :dataset_only_tree, :type => :boolean, :aliases => [ "--dataset_only_tree"], :default => false
-
+    method_option :data_query, :type => :string, :aliases => ["-q","--data_query"], :default => ""
 
     def remote_notebook()
       verify_logged_in(true)
@@ -3909,6 +4210,16 @@ module Cnvrg
         ds_sync_options = 1
       end
 
+      data_query = nil
+      if data.present?
+        data_query = options["data_query"]
+      end
+
+      if data_commit.present? and data_query.present?
+        log_message("Please use only one option: --query(-q) or --data_commit ", Thor::Shell::Color::RED)
+        exit(1)
+      end
+
       begin
         project = Project.new(working_dir)
         exp = Experiment.new(project.owner, project.slug)
@@ -3918,7 +4229,7 @@ module Cnvrg
         end
         invoke :sync, [false], []
         slug = ""
-        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options)
+        res = exp.remote_notebook(instance_type, commit, data, data_commit, notebook_type,ds_sync_options,data_query)
         if Cnvrg::CLI.is_response_success(res)
           slug = res["result"]["notebook_url"]
           log_message("#{Helpers.checkmark} Notebook is ready: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/notebook_sessions/show/#{slug}", Thor::Shell::Color::GREEN)
@@ -5593,7 +5904,7 @@ module Cnvrg
         begin
           $LOG.error message: e.message, type: "error"
           size = e.backtrace.size
-          min = 4
+          min = 10
           if min >= size
             min = size
           end
@@ -5702,7 +6013,7 @@ module Cnvrg
       end
 
       def should_update_version
-        res = Cnvrg::API.request("/cli/version", 'GET')
+        res = Cnvrg::API.request("cli/version", 'GET')
         if Cnvrg::CLI.is_response_success(res, false)
           updated_version = res["result"]["version"]
           if updated_version != Cnvrg::VERSION
@@ -5796,17 +6107,19 @@ module Cnvrg
           #   end
           # end
 
-        config = YAML.load_file(File.expand_path('~') + "/.cnvrg/config.yml")
-        version_date = config.to_h[:version_last_check]
-        if version_date.nil?
-          version_date = get_start_day()
-        end
-        next_day = get_start_day + 86399
-        if not (version_date..next_day).cover?(Time.now)
-          if should_update_version()
-            say "There is a new version, run gem update cnvrg", Thor::Shell::Color::BLUE
-          end
-        end
+        # config = YAML.load_file(File.expand_path('~') + "/.cnvrg/config.yml")
+        # version_date = config.to_h[:version_last_check]
+        # if version_date.nil?
+        #   version_date = get_start_day()
+        # end
+        #   next_day = get_start_day + 86399
+        #   version_date = version_date.to_i
+        #   next_day = next_day.to_i
+        # if not (version_date..next_day).cover?(Time.now)
+        #   if should_update_version()
+        #     say "There is a new version, run gem update cnvrg", Thor::Shell::Color::BLUE
+        #   end
+        # end
         if in_dir
           is_cnvrg = is_cnvrg_dir
           if !is_cnvrg