cnvrg 0.0.145 → 0.0.146

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b633d9c59001fa185be3d8c960e812a421ecef4e
-  data.tar.gz: bb5b2d41f8d4c4f8a95ce30334cc4e034c2e9bc4
+  metadata.gz: db813c4622a353a9e9d60678849c904d4268d28f
+  data.tar.gz: ecbc4ec2f9acc052ba1b61b18a88f14c15eb0312
 SHA512:
-  metadata.gz: 200acbdc094104d6666b5ad5b1715dd9d241e2c2cd7e3d70d77da0f6e672feb57305d54bd684e721141141201289b3b20aa993bab1bc04d43ea8465dac41eb8c
-  data.tar.gz: 4a79957cb3800306ca5e15cd849ebb02afefd11913eca6f9f7b0145d8133f635c111e6363fdbe0390241405874fc4465c8744afc704dd822517742502ecb8408
+  metadata.gz: fcaf8b6352e76e300d75ea4db289b3c58c8cfe785522baa100cafa271afea54e0987dea60ab79387e5552258f55209cb667cff2f840d61199d28b5a44c4d8e92
+  data.tar.gz: 42b8ba42106b5d6321c56cc94db3e434102f6323941a3084cbf8a5bbef9242f154fca542591c3fce1a83f568893f14142b61392b7848120d50c13299476f35a9

data/lib/cnvrg/cli.rb

@@ -18,6 +18,8 @@ require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
 require 'cnvrg/Images'
+require 'cnvrg/dataset'
+require 'cnvrg/datafiles'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
@@ -29,8 +31,12 @@ require 'fileutils'
 require 'zip'
 require 'active_support/all'
 require 'thor'
+require 'pathname'
+require 'cnvrg/data'
+
 
 # DEV VERSION
+
 #
 module Cnvrg
   class CLI < Thor
@@ -38,18 +44,6 @@ module Cnvrg
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
     IP="localhost"
     PORT=7654
-    desc '', ''
-
-    def printable_commands(all = true, subcommand = false)
-      (all ? all_commands : commands).map do |_, command|
-        next if command.hidden? or (command.description.empty? and command.usage.empty?)
-        item = []
-        item << banner(command, false, subcommand)
-        item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
-        item
-      end.compact
-    end
-
     class << self
       # Hackery.Take the run method away from Thor so that we can redefine it.
       def is_thor_reserved_word?(word, type)
@@ -57,11 +51,11 @@ module Cnvrg
         super
       end
     end
+    desc "data", "upload and manage datasets", :hide =>true
+    subcommand "data", Data
 
-    desc "", ""
-    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
-
-    def testt
+    desc "", "" , :hide => true
+    def test
       # image_settings = {
       #     'Image' => "cnvrg:latest",
       #     'User' => 'ds',
@@ -86,7 +80,6 @@ module Cnvrg
 
 
     desc 'version', 'Prints cnvrg current version'
-
     def version
       puts Cnvrg::VERSION
 
@@ -95,7 +88,6 @@ module Cnvrg
     map %w(-v --version) => :version
 
     desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
-
     def set_api_url(url)
       home_dir = File.expand_path('~')
       if !url.end_with? "/api"
@@ -136,7 +128,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def set_remote_api_url(owner, current_user, url)
       home_dir = File.expand_path('~')
@@ -181,27 +173,41 @@ module Cnvrg
         owners = result["owners"]
         urls = result["urls"]
         choose_owner = result["username"]
-
         if owners.empty?
         else
           owners << choose_owner
           chosen = false
           while !chosen
-            choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
+            owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
+            choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
+
+            if choose_owner =~ /[[:digit:]]/
+              ow_index = choose_owner.to_i-1
+              if ow_index<0 or ow_index >= owners.size
+                say "No such owner, please choose again", Thor::Shell::Color::BLUE
+                chosen = false
+                next
+              end
+              choose_owner = owners[choose_owner.to_i-1]
+              chosen = true
 
-            owners_lower = owners.map { |o| o.downcase }
-            ow_index = owners_lower.index(choose_owner.downcase)
-            if ow_index.nil?
-              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
             else
-              chosen = true
+
+              owners_lower = owners.map { |o| o.downcase }
+              ow_index = owners_lower.index(choose_owner.downcase)
+              if ow_index.nil?
+                say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+              else
+                chosen = true
+              end
             end
+
           end
 
 
         end
-        if set_owner(owner, username,urls[ow_index])
-          say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
+        if set_owner(choose_owner, result["username"], urls[ow_index])
+          say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
         else
           say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
         end
@@ -343,8 +349,8 @@ module Cnvrg
 
     ## Projects
     desc 'new', 'Create a new cnvrg project'
-    method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
-    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
+    method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
+    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
 
     def new(project_name)
       begin
@@ -468,8 +474,8 @@ module Cnvrg
 
     end
     desc 'link', 'Link current directory to a new cnvrg project'
-    method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
-    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
+    method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
 
     def link
       begin
@@ -550,6 +556,252 @@ module Cnvrg
         exit(1)
       end
     end
+
+    desc 'init_data', 'Init dataset directory', :hide =>true
+    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
+
+    def init_data
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        dataset_name =File.basename(Dir.getwd)
+        if File.directory?(Dir.getwd+"/.cnvrg")
+          config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
+          say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
+
+          exit(0)
+        end
+        say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
+
+        working_dir = Dir.getwd
+        owner = CLI.get_owner
+        if Dataset.init(owner,dataset_name, options["public"])
+          path = Dir.pwd
+          @dataset = Dataset.new(path)
+          @dataset.generate_idx()
+
+          url = @dataset.url
+          check = Helpers.checkmark
+          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
+          say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
+          log_end(0)
+
+        else
+          log_end(1, "can't create dataset")
+          @dataset.revert(working_dir) unless @dataset.nil?
+          say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
+          exit(0)
+        end
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+    desc 'upload_data', 'Upload data files', :hide=>true
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
+
+    def upload_data(sync=false,direct=false)
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+
+        @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
+        ignore = options[:ignore] || []
+        if !@dataset.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
+        result = @dataset.compare_idx(false)
+        commit = result["result"]["commit"]
+          if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
+            log_end(0)
+
+            say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+            exit(1)
+          end
+
+        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
+        result = result["result"]["tree"]
+        # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
+        #   all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
+        #   if all.size == 1
+        #     num = "conflict"
+        #   else
+        #     num =  "conflicts"
+        #   end
+        #   say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
+        #   say "#{all.join("\n")}"
+        #   say "Please fix #{num}, and retry", Thor::Shell::Color::RED
+        #   exit(1)
+        #
+        # end
+        check = Helpers.checkmark()
+
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_end(0)
+          say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+        if options["verbose"]
+          if update_total == 1
+            say "Updating #{update_total} file", Thor::Shell::Color::BLUE
+          else
+            say "Updating #{update_total} files", Thor::Shell::Color::BLUE
+          end
+        else
+          say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
+
+        end
+
+        # Start commit
+
+        commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
+
+        # upload / update
+        begin
+          (result["added"] + result["updated_on_local"]).each do |f|
+            absolute_path = "#{@dataset.local_path}/#{f}"
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            if File.directory?(absolute_path)
+              resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+              if resDir
+                update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+              if res
+                update_count += 1
+                successful_updates<< relative_path
+              else
+                @files.rollback_commit(commit_sha1)
+                log_end(1, "can't upload, Rolling Back all changes")
+                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+                exit(0)
+              end
+            end
+          end
+
+          # delete
+          deleted = update_deleted(result["deleted"])
+          deleted.each do |f|
+            relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
+            if relative_path.end_with?("/")
+              if @files.delete_dir(f, relative_path, commit_sha1)
+                # update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              if @files.delete_file(f, relative_path, commit_sha1)
+                # update_count += 1
+                successful_updates<< relative_path
+              end
+            end
+          end
+
+        rescue SignalException
+          log_end(-1)
+          @files.rollback_commit(commit_sha1)
+          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          log_end(-1, e.message)
+          @files.rollback_commit(commit_sha1)
+          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          exit(0)
+        end
+        if !result["deleted"].nil? and !result["deleted"].empty?
+          update_count +=  result["deleted"].size
+        end
+        if update_count == update_total
+          res = @files.end_commit(commit_sha1)
+          if (Cnvrg::CLI.is_response_success(res, false))
+            # save idx
+            begin
+              @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
+
+              @dataset.update_idx_with_commit!(commit_sha1)
+            rescue => e
+              log_end(-1, e.message)
+              @files.rollback_commit(commit_sha1)
+              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
+              exit(1)
+
+            end
+            if options["verbose"]
+              say "#{check} Done", Thor::Shell::Color::BLUE
+              if successful_updates.size >0
+                say "Updated:", Thor::Shell::Color::GREEN
+                suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+                say suc.join("\n"), Thor::Shell::Color::GREEN
+              end
+              if successful_deletions.size >0
+                say "Deleted:", Thor::Shell::Color::GREEN
+                del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
+                say del.join("\n"), Thor::Shell::Color::GREEN
+              end
+              say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
+            else
+              if (options["sync"] or sync) and direct
+                say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
+
+              else
+                say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
+
+              end
+
+            end
+
+            log_end(0)
+          else
+            @files.rollback_commit(commit_sha1)
+            log_end(1, "error. Rolling Back all changes")
+            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
+          end
+        else
+          log_end(1, "error. Rolling Back all changes")
+          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
+
+          @files.rollback_commit(commit_sha1)
+        end
+      rescue =>e
+
+        log_end(-1)
+
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      end
+
+    end
+    desc 'unlink','Unlink a project from current directory'
+    def create_volume
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      dataset_dir = is_cnvrg_dir(Dir.pwd)
+      @dataset = Dataset.new(dataset_dir)
+      @dataset.create_volume()
+
+    end
+
     desc 'unlink','Unlink a project from current directory'
     def unlink
       verify_logged_in(false)
@@ -733,7 +985,7 @@ module Cnvrg
         exit(1)
       end
     end
-    desc '', 's'
+    desc '', '' , :hide => true
     def revert_exp
       begin
         log_start(__method__, args, options)
@@ -754,12 +1006,12 @@ module Cnvrg
 
 
     desc 'upload', 'Upload updated files'
-    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
+    method_option :ignore, :type => :array, :aliases => ["-i"], :desc => "ignore following files"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
-    def upload(link=false, sync=false)
+    def upload(link=false, sync=false,direct=false)
 
       begin
         verify_logged_in(true)
@@ -807,9 +1059,11 @@ module Cnvrg
         #   exit(1)
         #
         # end
+        check = Helpers.checkmark()
+
         if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
           log_end(0)
-          say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
+          say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
           return true
         end
         update_count = 0
@@ -906,7 +1160,6 @@ module Cnvrg
             if image and image.is_docker
               image.update_image_activity(commit_sha1, nil)
             end
-            check = Helpers.checkmark()
 
             if options["verbose"]
               say "#{check} Done", Thor::Shell::Color::BLUE
@@ -922,10 +1175,10 @@ module Cnvrg
               end
               say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
             else
-              if !options["sync"] and !sync
-                # say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
+              if (options["sync"] or sync) and direct
+                say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
 
-              # else
+              else
                 say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
 
               end
@@ -940,7 +1193,7 @@ module Cnvrg
           end
         else
           log_end(1, "error. Rolling Back all changes")
-          say "Error occurd, \nRolling back", Thor::Shell::Color::BLUE
+          say "Error occurd, \nRolling back", Thor::Shell::Color::RED
 
           @files.rollback_commit(commit_sha1)
         end
@@ -948,7 +1201,7 @@ module Cnvrg
 
         log_end(-1)
 
-        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+        say "Error occurd, \nAborting", Thor::Shell::Color::RED
         @files.rollback_commit(commit_sha1)
         exit(1)
       rescue SignalException
@@ -963,9 +1216,9 @@ module Cnvrg
     end
 
     desc 'download', 'Download updated files'
-    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-    method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
 
     def download(sync=false)
       begin
@@ -1050,7 +1303,7 @@ module Cnvrg
             say successful_changes.join("\n"), Thor::Shell::Color::GREEN
             say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
           else
-            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN if !sync
+            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
           end
 
 
@@ -1163,10 +1416,11 @@ module Cnvrg
 
 
     desc 'sync', 'Sync with remote server'
-    method_option :new_branch, :type => :boolean, :aliases => ["--nb"], :desc => "create new branch of commits"
-    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
 
-    def sync
+    def sync(direct=true)
+      verify_logged_in(true) if direct
       if options["verbose"]
       say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
       else
@@ -1175,7 +1429,7 @@ module Cnvrg
 
 
       invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
-      invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
+      invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
 
 
     end
@@ -1227,7 +1481,7 @@ module Cnvrg
     end
 
 
-    desc '', ''
+    desc '', '' , :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t"], :default => ""
@@ -1244,7 +1498,8 @@ module Cnvrg
       memory_average = 0
       verify_logged_in(true)
       log_start(__method__, args, options)
-      working_dir = Dir.pwd
+      working_dir = is_cnvrg_dir
+      script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
 
 
       sync_before = options["sync_before"]
@@ -1268,7 +1523,7 @@ module Cnvrg
         else
           if sync_before
             # Sync before run
-            invoke :sync, [], :new_branch => is_new_branch
+            invoke :sync, [false], :new_branch => is_new_branch
           end
         end
         #set image for the project
@@ -1308,8 +1563,8 @@ module Cnvrg
         platform = RUBY_PLATFORM
         machine_name = Socket.gethostname
         begin
-          machine_activity = @exp.get_machine_activity(Dir.pwd)
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          machine_activity = @exp.get_machine_activity(working_dir)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
           unless @exp.slug.nil?
             real = Time.now
             exp_success = true
@@ -1418,8 +1673,9 @@ module Cnvrg
                 if sync_after
                   # Sync after run
 
+
                   download(sync=true)
-                  upload(link=false, sync=true)
+                  upload(link=false, sync=true,direct=false)
 
                 end
                 end_commit = @project.last_local_commit
@@ -1457,7 +1713,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
@@ -1642,7 +1898,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
     method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
     method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
     method_option :title, :type => :string, :aliases => ["-t"], :default => ""
@@ -1659,6 +1915,8 @@ module Cnvrg
       verify_logged_in(true)
       log_start(__method__, args, options)
       working_dir = is_cnvrg_dir
+      path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
+
       begin
         grid = options["grid"] || nil
 
@@ -1726,12 +1984,12 @@ module Cnvrg
 
 
 
-        invoke :sync, [], []
+        invoke :sync, [false], []
 
         say "Running remote experiment", Thor::Shell::Color::BLUE
         exp = Experiment.new(project.owner, project.slug)
 
-        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid)
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
         if Cnvrg::CLI.is_response_success(res)
 
           # if res["result"]["machine"] == -1
@@ -1831,7 +2089,7 @@ module Cnvrg
 
 
 
-        invoke :sync, [], []
+        invoke :sync, [false], []
 
         res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
 
@@ -1898,9 +2156,8 @@ module Cnvrg
     method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
     method_option :image, :type => :string, :aliases => ["--i"], :default => ""
 
-    desc 'starts a notebook session', 'starts a notebook session remotely or locally'
-
-    def notebook()
+    desc 'notebook', 'starts a notebook session remotely or locally'
+    def notebook
       local = options["local"]
       notebook_dir = options["notebook_dir"]
       kernel = options["kernel"]
@@ -1961,7 +2218,7 @@ module Cnvrg
           # end
         end
 
-        invoke :sync, [], []
+        invoke :sync, [false], []
 
 
 
@@ -2004,7 +2261,7 @@ module Cnvrg
             note_url = res["result"]["notebook_url"]
             @image.set_note_url(note_url)
             check = Helpers.checkmark()
-            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
+            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
             # Launchy.open(url)
 
             exit(0)
@@ -2209,7 +2466,7 @@ module Cnvrg
           end
 
         end
-        invoke :sync, [], :verbose => options["verbose"]
+        invoke :sync, [false], :verbose => options["verbose"]
         say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
         #replace url
         base_url = get_base_url()
@@ -2288,7 +2545,6 @@ module Cnvrg
     method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
     method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
-
     def notebook_stop
       begin
         verify_logged_in(true)
@@ -2299,7 +2555,7 @@ module Cnvrg
 
 
         say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
-        invoke :sync, [], :verbose => options["verbose"]
+        invoke :sync, [false], :verbose => options["verbose"]
 
         say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
 
@@ -2472,7 +2728,7 @@ module Cnvrg
       end
 
     end
-    desc 'run commands inside containers', 'run commands inside containers'
+    desc 'build', 'run commands inside containers'
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
 
     def build(*cmd)
@@ -2627,7 +2883,7 @@ module Cnvrg
 
     end
 
-    desc 'push image to cnvrg repository', 'push image to cnvrg repository'
+    desc 'push', 'push image to cnvrg repository'
 
     def push(*name)
       verify_logged_in(true)
@@ -2734,7 +2990,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def upload_log()
       log_path = '/home/ds/app/uwsgi.log'
@@ -2743,7 +2999,7 @@ module Cnvrg
 
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def exec_container(container_id, *cmd)
       container = Docker::Container.get(container_id)
@@ -2755,7 +3011,7 @@ module Cnvrg
 
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def port_container(container_id)
       container = Docker::Container.get(container_id)
@@ -2764,7 +3020,7 @@ module Cnvrg
 
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def stop_container(container_id)
       container = Docker::Container.get(container_id)
@@ -2773,7 +3029,7 @@ module Cnvrg
 
     end
 
-    desc '', ''
+    desc '', '', :hide => true
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
     method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
@@ -2836,7 +3092,7 @@ module Cnvrg
         return false
       end
     end
-    desc '', ''
+    desc '', '', :hide => true
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
     method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
     method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
@@ -2904,7 +3160,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
     method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
 
     def config_flask_remote(image_name, port=80)
@@ -2959,7 +3215,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def upload_cnvrg_image(image_name)
       verify_logged_in(false)
@@ -3020,7 +3276,7 @@ module Cnvrg
       end
     end
 
-    desc '', ''
+    desc '', '', :hide => true
 
     def download_image(image_name, image_slug)
       begin
@@ -3397,7 +3653,6 @@ module Cnvrg
       def self.is_response_success(response, should_exit=true)
         if response.nil?
           if !Cnvrg::Helpers.internet_connection?
-            # Cnvrg::CLI.log_end(1,"no internet connection")
             say("<%= color('Error:You seems to be offline', RED) %>")
           else
             say("<%= color('Error', RED) %>")
@@ -3543,11 +3798,15 @@ module Cnvrg
       def is_cnvrg_dir(dir=Dir.pwd)
         current_dir = dir
         home_dir = File.expand_path('~')
+        if current_dir.eql? home_dir
+          return false
+        end
         is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
         until is_cnvrg == true
           current_dir = File.expand_path("..", current_dir)
           is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
-          if File.expand_path("..", current_dir).eql? home_dir
+          if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
+            is_cnvrg = false
             break
           end
         end
@@ -3753,6 +4012,16 @@ module Cnvrg
 
         return final
       end
+      def get_cmd_path_in_dir(main_dir,sub_dir)
+        first = Pathname.new main_dir
+        second = Pathname.new sub_dir
+        relative = second.relative_path_from first
+        if relative.eql? "."
+          return ""
+        else
+          return relative
+        end
+      end