cnvrg 1.6.32 → 1.9.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ccd271bedf17f196897bdc28e238698676613ae09f391a9b412e3135703e3f7b
-  data.tar.gz: 3f61217606f6f9c50e0e27dee3c9f3f4d00d1eec7bd48622d4c62b4de5562f5f
+  metadata.gz: e708ef034df38ed0b4f5c1ac4bb02fa79a26c93b188f571256f75dbc9d2eaaa6
+  data.tar.gz: 6badf54b65660776e63c02c7d3c5dbbab83d0e1e83f6e877b48d77fad5ba3036
 SHA512:
-  metadata.gz: afd4299de4463f503d3e45528974ad451b941331af03cb067c6a801fdb25d49197ae5673505f8bd30f03c258026b63324b952d80270925040cf4cf344409a4ee
-  data.tar.gz: 6d1716507fdf9abbc1199a5815f653242bfc0f43bac340b018696eed869e22f6be89cf0ef1d27e1a44516541d69e2e1332b3eb5a6aba25f098504d97a0eb03f4
+  metadata.gz: 21d89ec4fb99c4102bc1e8e0e50df516339a1c9e9660ee8f0dd8acf3ae30bd27067f5ea4fe979de3b737bd6f748ced98f023100487a4226b7f21eed17975142c
+  data.tar.gz: 91fb2d10994c11e9b28ef3bbc128f847ac2efd641892c29ec1ec2b16d4b125266e85a6166153b66ab9e9e1c475190f6eca771e42d739a02c1136dbe8cb6c3abb

data/cnvrg.gemspec

@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
     spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
     spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
-    spec.add_runtime_dependency 'aws-sdk', '~> 2.11.417'
+    spec.add_runtime_dependency 'aws-sdk', '~> 3.0'
     spec.add_runtime_dependency 'signet', '~> 0.11.0'
     spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
     spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'
@@ -40,11 +40,8 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
     spec.add_runtime_dependency 'parallel', '~> 1.12.0'
     spec.add_runtime_dependency 'azure-storage-blob', '~> 1.1.0'
-
     spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
-    spec.add_runtime_dependency 'docker-api', '~> 1.33'
     spec.add_runtime_dependency 'activesupport', '~> 5.2.0'
     spec.add_runtime_dependency 'ruby-progressbar'
-    spec.add_runtime_dependency 'net-ssh'
     spec.add_runtime_dependency 'down'
 end

data/lib/cnvrg/Images.rb

@@ -1,7 +1,5 @@
 require 'fileutils'
 require 'cnvrg/files'
-require 'docker'
-require 'net/ssh'
 require 'mimemagic'
 
 
@@ -175,58 +173,6 @@ module Cnvrg
       response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'POST', {image_logs:logs})
       return response
     end
-    def self.ssh_to_machine(resp)
-
-      sts_path = resp["result"]["sts_path"]
-
-      uri = URI.parse(sts_path)
-
-      http_object = Net::HTTP.new(uri.host, uri.port)
-      http_object.use_ssl = true if uri.scheme == 'https'
-      request = Net::HTTP::Get.new(sts_path)
-
-      body = ""
-      http_object.start do |http|
-        response = http.request request
-        body = response.read_body
-      end
-
-      URLcrypt::key = [body].pack('H*')
-
-      ip = URLcrypt.decrypt(resp["result"]["machine_i"])
-
-      user = URLcrypt.decrypt(resp["result"]["machine_u"])
-      key = URLcrypt.decrypt(resp["result"]["machine_k"])
-      tempssh = Tempfile.new "sshkey"
-      tempssh.write open(key).read
-      tempssh.rewind
-      key_path = tempssh.path
-      count = 0
-      while count < 5
-
-      begin
-          ssh = Net::SSH.start(ip, user=user, :keys => key_path, :timeout => 10)
-          if !ssh.nil?
-            return ssh
-          else
-            count+=1
-            sleep(2)
-
-          end
-        rescue
-      count+=1
-      sleep(2)
-
-
-      end
-      end
-      if tempssh
-        tempssh.close
-        tempssh.unlink
-      end
-      return false
-    end
-
 
 
     def create_custom_image(new_image_name,working_dir,stored_commands)
@@ -270,100 +216,6 @@ module Cnvrg
       File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
     end
 
-    def get_container(stop=false)
-      begin
-        container_id=is_container_exist()
-
-        if !container_id
-          return create_container()
-        else
-          container = Docker::Container.get(container_id)
-          status = container.json["State"]["Status"]
-
-          if status == "running"
-            return container
-          else
-            if stop
-              return false
-            end
-            res = container.start()
-            if res.info["State"]["Status"].eql? "exited" and  res.info["State"]["Error"].include? "port is already allocated"
-              return create_container()
-            end
-            return container
-          end
-        end
-      rescue => e
-        if e.message.include? "No such container"
-
-          return create_container()
-        else
-          return false
-        end
-      end
-
-    end
-
-    def create_container(port=7654, is_remote=false)
-      begin
-        image_settings = {
-            'Image' => "#{@image_name}:latest",
-            'User' => 'ds',
-            'Cmd' => '/usr/local/cnvrg/run_ipython.sh',
-            'WorkingDir' => '/home/ds/notebooks',
-            'ExposedPorts' => {
-                '8888/tcp' => {},
-            },
-            'HostConfig' => {
-                'Binds' => ["#{@working_dir}:/home/ds/notebooks"],
-                'PortBindings' => {
-                    '8888/tcp' => [
-                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
-                    ],
-                },
-            },
-        }
-        container = Docker::Container.create(image_settings)
-        container.start()
-        netrc = File.open(File.expand_path('~')+"/.netrc", "rb")
-        netrc_content = netrc.read
-        container.store_file("/home/ds/.netrc", netrc_content)
-        command = ["/bin/bash", "-lc", "sudo chmod 600 /home/ds/.netrc"]
-        p = container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.netrc"]
-        p = container.exec(command, tty: true)
-        config = File.open(File.expand_path('~')+"/.cnvrg/config.yml", "rb")
-        config_content = config.read
-        container.store_file("/home/ds/.cnvrg/config.yml", config_content)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg"]
-        container.exec(command, tty: true)
-        # Libraries instlled
-        save_installed_libraries(container)
-        config = {project_name: @project_name,
-                  project_slug: @project_slug,
-                  owner: @owner,
-                  docker: true, image_base: @image_name, image_tag: @image_tag, container: container.id, port: port, image_slug: @image_slug}
-
-        File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-
-
-        return container
-
-
-      rescue => e
-        if e.message.include? "is not running"
-          return create_container(port-1)
-        end
-        return false
-      rescue SignalException
-
-        say "\nAborting", Thor::Shell::Color::RED
-        exit(1)
-      end
-
-
-    end
-
     def save_installed_libraries(container)
       begin
         command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']

data/lib/cnvrg/api.rb

@@ -77,20 +77,22 @@ module Cnvrg
                 if response.to_hash[:status] == 404
                   return false
                 end
-                if parse_request == true
+                if parse_request
                     JSON.parse(response.body)
                 else
                     response
                 end
-                when 'POST', 'PUT'
+            when 'POST', 'PUT'
                 conn.options.timeout = 4200
-                conn.options.open_timeout=180
+                conn.options.open_timeout = 180
+                conn.headers['Content-Type'] = "application/json"
                 retries = 0
                 success = false
+                data = data || {}
                 while !success and retries < 20
                     begin
-                        response = conn.post "#{resource}", data if method.eql? 'POST'
-                        response = conn.put "#{resource}", data if method.eql? 'PUT'
+                        response = conn.post "#{resource}", data.to_json if method.eql? 'POST'
+                        response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
                         success = true
                         Cnvrg::API.parse_version(response)
 
@@ -113,7 +115,7 @@ module Cnvrg
                 end
             when 'POST_JSON'
               conn.options.timeout = 4200
-              conn.options.open_timeout =4200
+              conn.options.open_timeout = 4200
               conn.headers['Content-Type'] = "application/json"
               new_data = JSON.dump(data)
 
@@ -124,8 +126,6 @@ module Cnvrg
                 begin
                   response = conn.post "#{resource}", new_data
                   success = true
-                  Cnvrg::API.parse_version(response)
-
                 rescue => e
                   Cnvrg::Logger.log_error(e)
                   sleep(5)

data/lib/cnvrg/api_v2.rb

@@ -0,0 +1,14 @@
+module Cnvrg
+  class API_V2 < API
+    ENDPOINT_VERSION = 'v2'
+
+    def self.endpoint_uri
+      api = get_api()
+      return "#{api}/#{Cnvrg::API_V2::ENDPOINT_VERSION}"
+    end
+
+    def self.is_response_success(response)
+      raise Exception.new("Bad status in response #{response.status}") if response.status != 200
+    end
+  end
+end

data/lib/cnvrg/cli.rb

@@ -12,7 +12,6 @@ require 'digest' # sha1up
 require "highline/import"
 require 'socket'
 require 'thor'
-require 'docker'
 require 'socket'
 require 'timeout'
 require 'fileutils'
@@ -28,13 +27,11 @@ require 'cnvrg/auth'
 require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
-require 'cnvrg/Images'
 require 'cnvrg/image'
 require 'cnvrg/dataset'
 require 'cnvrg/datafiles'
 require 'cnvrg/data'
 require 'cnvrg/storage'
-require 'cnvrg/ssh'
 require 'cnvrg/result'
 require 'cnvrg/logger'
 require 'cnvrg/org_helpers'
@@ -49,6 +46,9 @@ require 'cnvrg/downloader/clients/s3_client'
 require 'cnvrg/downloader/clients/gcp_client'
 require 'cnvrg/downloader/clients/azure_client'
 require 'cnvrg/job_cli'
+require 'cnvrg/job_ssh'
+require 'cnvrg/connect_job_ssh'
+require 'cnvrg/api_v2'
 
 class Thor
   module Base
@@ -175,6 +175,9 @@ module Cnvrg
     desc "job", "manage running jobs", :hide => false
     subcommand "job", JobCli
 
+    desc "ssh", "ssh into running jobs", :hide => false
+    subcommand "ssh", JobSsh
+
     desc "image [COMMAND]", "build existing images", :hide => true
     subcommand "image", ImageCli
 
@@ -819,9 +822,9 @@ module Cnvrg
     end
 
     desc 'data verify', 'Verify datasets', :hide => true
-    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => nil
 
-    def verify_datasets(dataset_titles, timeout=0)
+    def verify_datasets(dataset_titles, timeout=nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -830,21 +833,31 @@ module Cnvrg
         log_message("All datasets are verified", Thor::Shell::Color::BLUE) if verified
         log_message("Failed to verify datasets", Thor::Shell::Color::RED) if !verified
         exit(1) if !verified
-
       rescue SignalException
         say "\nAborting", Thor::Shell::Color::RED
         exit(1)
       end
     end
 
+    desc 'data scan', 'Lookup datasets', :hide => true
+    def scan_datasets()
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        log_message("Scanning datasets", Thor::Shell::Color::BLUE)
+        datasets = Dataset.scan_datasets()
+        puts(datasets.to_json)
+      end
+    end
+
     desc 'data clone', 'Clone dataset', :hide => true
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
     method_option :only_tree, :type => :boolean, :aliases => ["-t", "--tree"], :default => false
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
-
-    def clone_data(dataset_url,only_tree=false,commit=nil,query=nil,read=false,remote=false, relative: false)
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -853,10 +866,10 @@ module Cnvrg
         read = options["read"] || read || false
         remote = options["remote"] || remote || false
         query = options['query'].presence || query.presence
+        soft = options['soft'] || soft
         if query.present?
-          return clone_data_query(dataset_url, query)
+          return clone_data_query(dataset_url, query, flatten, soft: soft)
         end
-        @executer = Cnvrg::Helpers::Executer.get_executer
 
         url_parts = dataset_url.split("/")
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
@@ -868,6 +881,8 @@ module Cnvrg
         dataset_name = response["result"]["name"]
         dataset_home = Dir.pwd+"/"+dataset_name
 
+        Dataset.stop_if_dataset_present(dataset_home, dataset_name, commit: response["result"]["commit"]) if soft
+
         check = Helpers.checkmark
         if @dataset.init_home(remote:remote)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
@@ -875,14 +890,12 @@ module Cnvrg
           log_message("Downloading files", Thor::Shell::Color::BLUE)
           if @dataset.softlinked?
             @files.cp_ds(relative: relative)
-            @executer.set_dataset_status(dataset: @dataset.slug, status: "cloned") if @executer
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
             @dataset.write_success
             return
           end
 
           if only_tree
-
             success = Dataset.clone_tree(commit: commit, dataset_home: dataset_home)
             return if success
           end
@@ -900,7 +913,7 @@ module Cnvrg
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
-            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read)
+            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten)
 
             downloaded_files += files['keys'].length
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -908,7 +921,6 @@ module Cnvrg
           progressbar.finish
           if downloaded_files == files_count
             Dataset.verify_cnvrgignore_exist(dataset_name, false)
-            @executer.set_dataset_status(dataset: @dataset.slug, status: "cloned") if @executer
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
             @dataset.write_success
             ### if read, dont generate idx (but create idx.yml) if not read, generate idx.
@@ -930,12 +942,14 @@ module Cnvrg
 
     desc 'data clone_query', 'Clone dataset _query', :hide => true
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => ""
-    def clone_data_query(dataset_url,query=nil)
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    def clone_data_query(dataset_url, query=nil, flatten=false, soft: false)
       begin
         verify_logged_in(false)
-        @executer = Cnvrg::Helpers::Executer.get_executer
+        #@executer = Cnvrg::Helpers::Executer.get_executer
         log_start(__method__, args, options)
         query = options["query"] || query
+        soft = options["soft"] || soft
         if !query.present?
           log_message("Argument missing : query", Thor::Shell::Color::RED)
           exit(1)
@@ -945,13 +959,14 @@ module Cnvrg
         project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
-
         response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/search/#{query}", 'GET')
         Cnvrg::CLI.is_response_success(response,true)
         dataset_name = response["results"]["name"]
         dataset_slug = response["results"]["slug"]
-        dataset_home = File.join(Dir.pwd, dataset_name)
+        dataset_home = Dir.pwd+"/"+dataset_slug
+        Dataset.stop_if_dataset_present(dataset_home, dataset_name) if soft
 
+        # dataset_home = Dir.pwd
         if Dataset.blank_clone(owner, dataset_name, dataset_slug)
           dataset = Dataset.new(dataset_home)
           log_message("Cloning #{dataset_name}", Thor::Shell::Color::BLUE)
@@ -966,6 +981,7 @@ module Cnvrg
               },
               in_threads: ParallelThreads
           }
+
           begin
             log_message("Downloading files", Thor::Shell::Color::BLUE)
             Parallel.map((response["results"]["query_files"]), parallel_options) do |f|
@@ -974,6 +990,7 @@ module Cnvrg
               file_name = relative_path_dir.pop()
               relative_path_dir = relative_path_dir.join("/")
               abs_path = dataset_home + "/" + relative_path_dir
+              abs_path = dataset_home if flatten
               begin
                 FileUtils.mkdir_p(abs_path) unless File.exist? (abs_path + "/" + file_name)
               rescue
@@ -981,14 +998,14 @@ module Cnvrg
                 exit(1)
               end
               begin
-                File.write "#{abs_path}/#{file_name}", open(f["s3_url"]).read unless File.exist? (abs_path + "/" + file_name)
-              rescue
+                File.write "#{abs_path}/#{file_name}", open(f["url"]).read unless File.exist? (abs_path + "/" + file_name)
+              rescue => e
                 log_message("Could not download file: #{f["fullpath"]}", Thor::Shell::Color::RED)
                 exit(1)
               end
 
             end
-            @executer.set_dataset_status(dataset: dataset.slug, status: "cloned") if @executer.present?
+            #@executer.set_dataset_status(dataset: dataset.slug, status: "cloned") if @executer.present?
           rescue Interrupt
             log_message("Couldn't download", Thor::Shell::Color::RED)
             exit(1)
@@ -998,7 +1015,7 @@ module Cnvrg
             check = Helpers.checkmark
             log_message("#{check} Clone finished successfully", Thor::Shell::Color::GREEN)
             dataset.write_success(in_folder=true)
-          rescue
+          rescue => e
               exit(1)
           end
         end
@@ -1008,32 +1025,6 @@ module Cnvrg
       end
     end
 
-    desc 'init_data_container', 'Init dataset directory', :hide => true
-    method_option :login_content, :type => :string, :aliases => ["-l"], :default => ""
-
-    def init_data_container(container)
-      begin
-        login_content = options["login_content"]
-
-        container = Docker::Container.get(container)
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-
-      rescue SignalException
-
-        say "\nAborting", Thor::Shell::Color::RED
-        exit(1)
-      end
-    end
-
     desc 'data_snap', 'Init dataset directory', :hide => true
     method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
 
@@ -1184,17 +1175,29 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
+    def get_owner_slug(url_or_slug)
+      if url_or_slug =~ URI::regexp
+        # Find owner and slug in url
+        url_parts = url_or_slug.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(url_or_slug, "datasets")
+        slug = url_parts[project_index + 1]
+        owner = url_parts[project_index - 1]
+      else
+        # Find owner in config file
+        owner = CLI.get_owner
+        slug = url_or_slug
+      end
+      return owner, slug
+    end
+
+    desc '', '', :hide => true
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, threads: 15, message: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
 
-        #find owner and slug in url
-        url_parts = dataset_url.split("/")
-        project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
-        slug = url_parts[project_index + 1]
-        owner = url_parts[project_index - 1]
-        @dataset = Dataset.new(dataset_url: dataset_url)
+        owner, slug = get_owner_slug(dataset_url)
+        @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
         @files = @datafiles.verify_files_exists(files)
 
@@ -1204,7 +1207,7 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
-          response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
+          response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
@@ -1218,28 +1221,33 @@ module Cnvrg
         else
           @commit = commit
         end
-        #dir shouldnt have starting or ending slash.
+
+        # dir shouldnt have starting or ending slash.
         dir = dir[0..-2] if dir.end_with? '/'
         dir = dir[1..-1] if dir.start_with? '/'
 
-        @files.each_slice(chunk_size).each do |list_files|
-          temp_tree = @dataset.generate_chunked_idx(list_files, prefix: dir)
-          #will throw a signal exception if something goes wrong.
-          @datafiles.upload_multiple_files(@commit, temp_tree, force: true, prefix: dir, total: @files.size)
+        @datafiles.upload_multiple_files_optimized(
+          @files,
+          @commit,
+          force: force,
+          chunk_size: chunk_size,
+          prefix: dir,
+          threads: threads
+        )
+
+        # This is for backwards compatibility only and should be removed in future versions:
+        res = @datafiles.put_commit(@commit)
+        unless res.is_success?
+          raise SignalException.new(1, res.msg)
         end
-        if commit.blank?
-          res = @datafiles.put_commit(@commit)
-          unless res.is_success?
-            raise SignalException.new(1, res.msg)
-          end
-        else
-          res = @datafiles.end_commit(@commit,false, success: true )
-          msg = res['result']
-          response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
-          unless response.is_success?
-            raise SignalException.new(1, res.msg)
-          end
+
+        res = @datafiles.end_commit(@commit,false, success: true, commit_type: "put")
+        msg = res['result']
+        response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
+        unless response.is_success?
+          raise SignalException.new(1, res.msg)
         end
+
         log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
@@ -1248,7 +1256,49 @@ module Cnvrg
     end
 
 
+    desc '', '', :hide => true
+    def data_rm(dataset_url, regex_list: [], commit: '', message: nil)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+
+        owner, slug = get_owner_slug(dataset_url)
+        @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
+        @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
+
+        # Init a new commit
+        response = @datafiles.start_commit(false, true, chunks: 1, message: message )
+        unless response #means we failed in the start commit.
+          raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
+        end
+        @commit =  response['result']['commit_sha1']
+        files_to_delete, folders_to_delete, job_id = @datafiles.delete_multiple_files(@commit, regex_list)
+        log_message("Deleting #{files_to_delete} files and #{folders_to_delete} folders", Thor::Shell::Color::GREEN)
 
+        total_files = files_to_delete + folders_to_delete
+        current_progress = 0
+        progressbar = @datafiles.create_progressbar("Delete Progress", total_files)
+        chunk_size = 1000
+        offset = 0
+        while current_progress < total_files
+          current_progress = @datafiles.delete_file_chunk(@commit, regex_list, chunk_size, offset)
+          progressbar.progress = current_progress
+          offset += chunk_size
+        end
+
+        res = @datafiles.end_commit(@commit,false, success: true)
+        msg = res['result']
+        response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
+        unless response.is_success?
+          raise SignalException.new(1, res.msg)
+        end
+
+        log_message("Deleting files finished Successfully", Thor::Shell::Color::GREEN)
+      rescue SignalException => e
+        log_message(e.message, Thor::Shell::Color::RED)
+        return false
+      end
+    end
 
     desc 'upload_data', 'Upload data files', :hide => true
     method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
@@ -1699,18 +1749,22 @@ module Cnvrg
     end
 
     desc 'data commits', 'List all commits for a specific dataset', :hide => true
-
-    def list_dataset_commits()
-      verify_logged_in(true)
+    def list_dataset_commits(dataset_url, commit_sha1: nil)
+      verify_logged_in(false)
       log_start(__method__, args, options)
 
-      dataset_dir = is_cnvrg_dir(Dir.pwd)
-      @dataset = Dataset.new(dataset_dir)
-      result = @dataset.list_commits()
+      if dataset_url == "."
+        dataset_dir = is_cnvrg_dir(Dir.pwd)
+        @dataset = Dataset.new(dataset_dir)
+      else
+        owner, slug = get_owner_slug(dataset_url)
+        @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
+      end
+
+      result = @dataset.list_commits(commit_sha1:commit_sha1)
       list = result["result"]["list"]
 
       print_table(list)
-
     end
 
     desc 'commits', 'List all commits for a specific Project'
@@ -1741,17 +1795,17 @@ module Cnvrg
 
 
     desc 'git_clone', 'Clone project'
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     def git_clone(slug, owner)
       verify_logged_in(false)
       log_start(__method__, args, options)
-
+      project_home = Dir.pwd
+      soft = options["soft"] || false
+      Project.stop_if_project_present(project_home, slug) if soft
       clone_resp = Project.clone_dir_remote(slug, owner, slug,true)
-      idx_status = Project.new(get_project_home).generate_idx
+      exit 1 if not clone_resp
+      idx_status = Project.new(get_project_home).generate_idx(files:[])
       FileUtils.mkdir_p File.join(get_project_home, ENV['CNVRG_OUTPUT_DIR']) if ENV['CNVRG_OUTPUT_DIR'].present?
-      @executer = Cnvrg::Helpers::Executer.get_executer
-      if @executer.present?
-        @executer.update_git_commit
-      end
     end
 
 
@@ -1791,7 +1845,7 @@ module Cnvrg
     desc 'clone PROJECT_URL', 'Clone project'
     method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
     method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
-
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     def clone(project_url)
       begin
         verify_logged_in(false)
@@ -1801,6 +1855,8 @@ module Cnvrg
         slug = url_parts[project_index + 1]
         owner = url_parts[project_index - 1]
         remote = options["remote"] || false
+        soft = options["soft"] || false
+
 
         response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
         Cnvrg::CLI.is_response_success(response)
@@ -1814,6 +1870,8 @@ module Cnvrg
         clone_resp = false
         project_home = Dir.pwd
 
+        Project.stop_if_project_present(project_home, project_name) if soft
+
         if remote and !git
           clone_resp = Project.clone_dir_remote(slug, owner, project_name,git)
         elsif git
@@ -1837,8 +1895,6 @@ module Cnvrg
           end
           clone_resp = Project.clone_dir(slug, owner, project_name,git)
           project_home = Dir.pwd + "/" + project_name
-
-
         end
 
         if clone_resp
@@ -1956,8 +2012,6 @@ module Cnvrg
     method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
     method_option :init, :type => :boolean, :aliases => ["--initial"], :desc => "initial sync", :default => false
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
-
-
     def sync_data_new(new_branch, force, verbose, commit, all_files, tags ,parallel, chunk_size, init, message)
       verify_logged_in(true)
       log_start(__method__, args, options)
@@ -1966,11 +2020,13 @@ module Cnvrg
         # w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
         total_deleted, total_downloaded = invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
       end
-      # w(new_branch,  verbose,sync,force, tags, chunk_size)
+
       invoke :upload_data_new,[new_branch, verbose, true, force, tags, chunk_size, message:message, total_deleted: total_deleted, total_downloaded: total_downloaded],
              :new_branch=>new_branch, :direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel, :message => message
 
     end
+
+
     desc 'upload_data_new', 'upload_data_new', :hide => true
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -2213,15 +2269,27 @@ module Cnvrg
     method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
     method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
     method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
+    method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
+    method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
 
-    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil)
+    def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true)
       begin
         # we are passing "force" twice.. doesnt really make sense :\\
         verify_logged_in(true)
         log_start(__method__, args, options)
         @project = Project.new(get_project_home)
+
+        # Enable local/experiment exception logging
+        suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
+        if in_exp
+          exp_obj = Experiment.new(@project.owner, @project.slug, job_id: job_slug)
+        else
+          exp_obj = nil
+        end
+
         commit_msg = options["message"]
         if commit_msg.nil? or commit_msg.empty?
           commit_msg = ""
@@ -2237,19 +2305,21 @@ module Cnvrg
           spec_files_to_upload = spec_files_to_upload.split(",")
         end
         if @project.is_git
+          list = []
           git_output_dir = options["output_dir"] || output_dir
           if git_output_dir.present?
             if git_output_dir.ends_with? "/"
               git_output_dir = git_output_dir[0..-2]
             end
             list = @project.generate_output_dir(git_output_dir)
-            spec_files_to_upload = list
-            if spec_files_to_upload.blank?
-              log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
-              return true
-            end
-            force = true
           end
+          list += @project.generate_git_diff if options["git_diff"]
+          spec_files_to_upload = list
+          if spec_files_to_upload.blank?
+            log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
+            return true
+          end
+          force = true
         end
 
         if ignore.nil? or ignore.empty?
@@ -2291,8 +2361,6 @@ module Cnvrg
         end
         update_count = 0
         update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
-        successful_updates = []
-        successful_deletions = []
         if options["verbose"]
           if update_total == 1
             log_message("Updating #{update_total} file", Thor::Shell::Color::BLUE)
@@ -2312,8 +2380,11 @@ module Cnvrg
         end
         job_type = options['job_type'] || job_type
         job_slug = options['job_slug'] || job_slug
-        commit_sha1 = @files.start_commit(new_branch, force: force, exp_start_commit: exp_start_commit,
-                                          job_type: job_type, job_slug: job_slug, start_commit: current_commit, message: options["message"])["result"]["commit_sha1"]
+        commit_sha1 = @files.start_commit(
+            new_branch, force: force, exp_start_commit: exp_start_commit,
+            job_type: job_type, job_slug: job_slug, start_commit: current_commit,message: options["message"],
+            debug_mode: options["debug_mode"]
+        )["result"]["commit_sha1"]
         # upload / update
         # delete
         to_upload = result["added"] + result["updated_on_local"]
@@ -2324,32 +2395,30 @@ module Cnvrg
                                          :starting_at => 0,
                                          :total => (to_upload.size + deleted.size),
                                          :autofinish => true)
-        @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
 
-        @files.delete_files_from_server(deleted, commit_sha1)
+        buffered_errors = @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions)
+        @files.delete_files_from_server(deleted, commit_sha1, suppress_exceptions: suppress_exceptions)
 
         progressbar.finish
+
+        if buffered_errors.is_a?(Hash)
+          buffered_errors.keys.each do |file|
+            to_upload.delete(file)
+            Cnvrg::CLI.log_message(buffered_errors[file], 'red')
+            exp_obj.job_log([buffered_errors[file]]) unless exp_obj.nil?
+          end
+        end
+
         res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
         unless Cnvrg::CLI.is_response_success(res, false)
           raise StandardError.new("Cant end commit")
         end
+
         # save idx
         @project.update_idx_with_files_commits!((to_upload + deleted), res["result"]["commit_time"])
         @project.update_idx_with_commit!(commit_sha1)
         if options["verbose"]
           log_message("#{check} Done", Thor::Shell::Color::BLUE)
-          if successful_updates.size > 0
-            successful_updates.flatten!
-            log_message("Updated:", Thor::Shell::Color::GREEN)
-            suc = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-            log_message(suc.join("\n"), Thor::Shell::Color::GREEN)
-          end
-          if successful_deletions.size > 0
-            successful_deletions.flatten!
-            log_message("Deleted:", Thor::Shell::Color::GREEN)
-            del = successful_updates.map {|x| x = Helpers.checkmark() + " " + x}
-            log_message(del.join("\n"), Thor::Shell::Color::GREEN)
-          end
           log_message("Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN)
         else
           if return_id
@@ -2374,9 +2443,13 @@ module Cnvrg
         if e.is_a? SignalException
           say "\nAborting", Thor::Shell::Color::BLUE
           say "\nRolling back all changes", Thor::Shell::Color::BLUE
+
+          exp_obj.job_log(["Aborting", "Rolling back all changes"])  unless exp_obj.nil?
         else
           log_message(error_message, Thor::Shell::Color::RED)
           log_error(e)
+
+          exp_obj.job_log([error_message, e])  unless exp_obj.nil?
         end
         @files.rollback_commit(commit_sha1) unless commit_sha1.nil?
         print_res = {
@@ -2894,6 +2967,11 @@ module Cnvrg
     method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
     method_option :files, :type => :string, :aliases => ["--files"], :default => nil
     method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
+    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
+    method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
+    method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
+    method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
+
     def sync(direct = true)
       verify_logged_in(true) if direct
       @project = Project.new(get_project_home)
@@ -2905,16 +2983,20 @@ module Cnvrg
       is_git = ENV['CNVRG_GIT_PROJECT'] == "true" || @project.is_git
       in_exp = options["in_exp"] || (job_slug.present? and job_type.present?)
       in_exp = false if job_type.present? and job_type == "NotebookSession"
+      output_dir = options["output_dir"] || ENV['CNVRG_OUTPUT_DIR']
+
       run_download = true
-      if options[:force] or options[:files].present? or options[:output_dir].present? or in_exp or @project.is_branch
+      if (job_type == "NotebookSession" and is_git) or job_type == "Experiment" or options['force']
         run_download = false
       end
-      if run_download
+
+      if run_download or options['debug_mode']
         invoke :download, [true, "", in_exp ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
       end
-      invoke :upload, [false, true,  direct, "",in_exp,options[:force],  options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
+      invoke :upload, [false, true,  direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
              :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
-             :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
+             :files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"], :debug_mode => options['debug_mode'], :git_diff => options["git_diff"]
+
     end
 
     desc 'run cmd', 'Runs an experiment'
@@ -3059,6 +3141,8 @@ module Cnvrg
     method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
     method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
     method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
+    method_option :docker_id, :type => :string, :aliases => ["--docker_id"], :desc => "docker id to watch", :default => ""
+    method_option :gpu_util_from_docker, :type => :boolean, :aliases => ["--gpu-util-from-docker"], :desc => "take gpu utilization from job docker", :default => false
     method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
     method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
     method_option :force, :type => :boolean, :aliases => ["-f", "--force"], :default => false
@@ -3066,6 +3150,7 @@ module Cnvrg
     method_option :periodic_sync, :type => :string, :aliases => ["-ps", "--periodic_sync"], :default => ""
     method_option :output_dir, :type => :string, :aliases => ["-o", "--output_dir"], :default => nil
     method_option :data_query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :use_bash, :type => :boolean, :aliases => ["-b", "--use_bash"], :default => false
 
     def exec(*cmd)
       log = []
@@ -3132,8 +3217,12 @@ module Cnvrg
         end
         remote = options["remote"]
         if remote
-          docker_id = `cat /etc/hostname`
-          docker_id = docker_id.strip()
+          if options["docker_id"].present?
+            docker_id = options["docker_id"]
+          else
+            docker_id = `cat /etc/hostname`
+            docker_id = docker_id.strip()
+          end
         end
         is_on_gpu = options["gpu"]
         start_commit = @project.last_local_commit
@@ -3143,9 +3232,9 @@ module Cnvrg
 
         platform = RUBY_PLATFORM
         machine_name = Socket.gethostname
+        machine_activity_slug = ENV["CNVRG_MACHINE_ACTIVITY"]
         begin
-          machine_activity = @exp.get_machine_activity(working_dir)
-          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path, sync_before_terminate, periodic_sync)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity_slug, script_path, sync_before_terminate, periodic_sync)
           log_message("Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN)
           log_message("Running: #{cmd}\n", Thor::Shell::Color::BLUE)
           unless @exp.slug.nil?
@@ -3163,7 +3252,7 @@ module Cnvrg
                   begin
                     stats = remote ? usage_metrics_in_docker(docker_id) : Helpers.ubuntu? ? {memory: memory_usage, cpu: cpu_usage} : {}
                     if is_on_gpu
-                      gu = gpu_util
+                      gu = gpu_util(take_from_docker: options["gpu_util_from_docker"], docker_id: docker_id)
                       stats['gpu_util'] = gu[0]
                       stats['gpu'] = gu[1]
                     end
@@ -3175,6 +3264,16 @@ module Cnvrg
                 end
               end
               start_time = Time.now
+              shell_type = options["use_bash"] ? "bash -l" : "sh"
+              if @exp.get_cmd.present?
+                cmd = @exp.get_cmd
+                if options["docker_id"].present? # Escape for docker exec
+                  cmd = cmd.gsub("\"", "\\\"")
+                end
+              end
+              if options["docker_id"].present?
+                  cmd = "docker exec -it #{options["docker_id"]} #{shell_type} -c \"#{cmd}\""
+              end
               PTY.spawn(@exp.as_env, cmd) do |stdout, stdin, pid, stderr|
                 begin
                   stdout.each do |line|
@@ -3189,7 +3288,7 @@ module Cnvrg
                       puts line
                     end
                     log << cur_log
-                    if log.size >= 5
+                    if log.size >= 1
                       @exp.upload_temp_log(log) unless log.empty?
                       log = []
                       elsif (start_time + 15.seconds) <= Time.now
@@ -3239,29 +3338,26 @@ module Cnvrg
                 exp_success = false
               end
 
-                if sync_after
-                  @exp.job_log(["Syncing Experiment"])
-                # Sync after run
-                  if @project.is_git
-                    output_dir = output_dir || @exp.output_dir
-                    if output_dir.present?
-                      upload(false, false, true, ignore, true, true,output_dir,"Experiment",@exp.slug  )
-                      # invoke :upload, [false, false, true, ignore, true, true], :output_dir => output_dir, :force=>true, :job_type=>'Experiment', :job_slug=>@exp.slug
-                    end
-                  else
-                    upload(false, false, true, ignore, true, true,nil,"Experiment",@exp.slug  )
-
-                    # invoke :upload, [false, false, true, ignore,true, true], :job_type=>'Experiment', :job_slug=>@exp.slug, :force=>true
+              if sync_after
+                @exp.job_log(["Syncing Experiment"])
+              # Sync after run
+                if @project.is_git
+                  output_dir = output_dir || @exp.output_dir
+                  if output_dir.present?
+                    upload(false, false, true, ignore, true, true, output_dir, "Experiment", @exp.slug, true )
                   end
-
+                else
+                  upload(false, false, true, ignore, true, true, nil, "Experiment", @exp.slug, true )
                 end
+              end
+
               end_commit = @project.last_local_commit
               if end_commit.present?
                 @exp.job_log(["Experiment end commit: #{end_commit}"])
               end
 
                 # log_thread.join
-                stats_thread.join
+              stats_thread.join
 
                 res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average, end_time: end_time)
 
@@ -3409,8 +3505,8 @@ module Cnvrg
         local_folders_options = options["local_folders"]
         options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
                              "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
-                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
-                             "email_notification_error", "email_notification_success", "emails")
+                             "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
+                             "requirements", "prerun", "email_notification_error", "email_notification_success", "emails")
         exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
         command = "#{exec_options} #{remote}  #{upload_output_option} #{cmd.flatten.join(" ")}"
         commit_to_run = options["commit"] || nil
@@ -4235,144 +4331,6 @@ module Cnvrg
 
     end
 
-    method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
-    method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
-    method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
-    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
-    method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
-    method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
-    method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
-    method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
-    method_option :base, :type => :boolean, :aliases => ["-b", "--base"], :default => false
-    method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
-    method_option :docker_path, :type => :string, :aliases => ["--docker_path"], :default => ""
-
-
-    desc 'create_custom_image', 'run commands inside containers', :hide => true
-
-    def build_image(image_name)
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
-                     "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
-        instance_type = get_instance_type(instances)
-        image_extend = options["image"]
-        public = options["public"]
-        base = options["base"]
-        python3 = options["python3"]
-        docker_path = options["docker_path"]
-        owner = CLI.get_owner
-        checks = Helpers.checkmark()
-        tar_path = nil
-        if !docker_path.nil? and !docker_path.empty?
-          docker_path = File.absolute_path(docker_path)
-          #create tar of the docker path: it could be a docker file, and it could be a docker folder
-          tar_path = File.expand_path('~') + "/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
-          resp = create_docker_tar(docker_path, tar_path)
-          if !resp
-            log_message("Couldn't create tar from docker path", Thor::Shell::Color::RED)
-            FileUtils.rm_rf tar_path
-            exit(1)
-          end
-          files = Cnvrg::Files.new(owner, "")
-          resp = Images.create_new_custom_image_with_docker(instance_type, owner, image_name, public, base, image_extend, python3, tar_path, files)
-          if resp
-          end
-        else
-          log_message("Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE)
-          resp = Images.create_new_custom_image(instance_type, owner, image_name, public, base, image_extend, python3, nil)
-
-        end
-
-        if Cnvrg::CLI.is_response_success(resp, false)
-          image_slug = resp["result"]["slug"]
-          container = resp["result"]["machine_c"]
-          log_message("#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN)
-          log_message("Connecting to machine", Thor::Shell::Color::BLUE)
-          ssh = Ssh.new(resp)
-          if !ssh.is_ssh
-            log_message("Couldn't connect to machine,aborting", Thor::Shell::Color::RED)
-            Images.revoke_custom_new_image(owner, image_slug)
-          end
-          log_message("run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE)
-          begin
-            logs = []
-
-            while true
-              command = ask("$>")
-              logs << {time: Time.now,
-                       message: command,
-                       type: "stdout"
-              }
-              if command.eql? "quit"
-                log_message("Commiting Image..", Thor::Shell::Color::BLUE)
-                break
-              end
-              res = ssh.exec_command(command)
-              begin
-                res_parsed = JSON.parse(res)
-                res = res_parsed.join(",")
-              end
-
-              puts res
-              logs << {time: Time.now,
-                       message: res,
-                       type: "stdout"
-              }
-              logs.flatten!
-
-            end
-
-          rescue SignalException
-            log_message("Commiting Image..", Thor::Shell::Color::BLUE)
-
-          end
-          resp = Images.commit_custom_image(owner, image_slug, logs)
-          if Cnvrg::CLI.is_response_success(resp, false)
-            log_message("#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN)
-          else
-            if image_slug
-              Images.revoke_custom_new_image(owner, image_slug)
-            end
-            if ssh
-              ssh.close_ssh()
-            end
-            log_message("Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED)
-
-            exit(1)
-          end
-          if ssh
-            ssh.close_ssh()
-          end
-
-
-        end
-      rescue => e
-        log_message("Error occurd, aborting", Thor::Shell::Color::RED)
-
-        log_error(e)
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh()
-        end
-
-
-      rescue SignalException
-        if image_slug
-          Images.revoke_custom_new_image(owner, image_slug)
-        end
-        if ssh
-          ssh.close_ssh
-        end
-        say "\nAborting"
-        exit(1)
-      end
-
-    end
-
 
     desc 'build', 'run commands inside containers', :hide => true
     method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
@@ -4566,66 +4524,7 @@ module Cnvrg
     end
 
 
-    desc 'upload_image', 'commit notebook changes to create a new notebook image', :hide =>true
-
-    def upload_image_old(image_id, is_public, is_base, *message)
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-      image = Docker::Image.get(image_id)
-      project_home = get_project_home
-      @project = Project.new(project_home)
-      last_local_commit = @project.last_local_commit
-      image_name = @project.slug + "#{last_local_commit}"
-      path = File.expand_path('~') + "/.cnvrg/tmp/#{image_name}.tar"
-      owner = Cnvrg::CLI.get_owner()
-      if !message.nil? or !message.empty?
-        message = message.join(" ")
-      end
-
-      log_message("Saving image's current state", Thor::Shell::Color::BLUE)
-      image.save(path)
-
-      begin
-        log_message("Compressing image file to upload", Thor::Shell::Color::BLUE)
-        gzipRes = system("gzip -f #{path}")
-        if !gzipRes
-
-          log_message("Couldn't create tar file from image", Thor::Shell::Color::RED)
-          exit(1)
-        end
-        path = path + ".gz"
-        @files = Cnvrg::Files.new(owner, "")
-
-        exit_status = $?.exitstatus
-        if exit_status == 0
-          log_message("Uploading image file", Thor::Shell::Color::BLUE)
-
-          diff = container_changes(Dir.pwd)
-          res = @files.upload_image(path, image_name, owner, is_public, is_base, diff[1], diff[0], diff[2], message, image.commit_id)
-          if res
-            File.delete(path)
-            image_loc = is_project_with_docker(Dir.pwd)
-            image_loc.update_slug(res["result"]["id"])
-
-            checks = Helpers.checkmark()
-            log_message("#{checks} Done", Thor::Shell::Color::GREEN)
-          else
-            log_message("Couldn't upload image", Thor::Shell::Color::RED)
-
-          end
-        else
-          log_message("Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED)
-          exit(1)
-        end
-      rescue => e
-        log_message("Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED)
-        log_error(e)
-      rescue SignalException
 
-        say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
-        exit(1)
-      end
-    end
 
     desc '', '', :hide => true
 
@@ -4636,278 +4535,30 @@ module Cnvrg
 
     end
 
-    desc '', '', :hide => true
-
-    def exec_container(container_id, *cmd)
-      container = Docker::Container.get(container_id)
-      container.start()
-      cnvrg_command = cmd.join(" ")
-      command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
-      res = container.exec(command, tty: true, wait: 5400)[0]
-      say res
-    end
-
-    desc '', '', :hide => true
-
-    def port_container(container_id)
-      container = Docker::Container.get(container_id)
-      say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
-    end
-
-    desc '', '', :hide => true
-
-    def tensor_port_container(container_id)
-      container = Docker::Container.get(container_id)
-      say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
-    end
-
-    desc '', '', :hide => true
-
-    def stop_container(container_id)
-      container = Docker::Container.get(container_id)
-      container.stop()
-      container.remove()
-
-    end
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c"], :default => "/usr/local/cnvrg/run_ipython.sh"
-
-
-    def config_remote(image_name, port = 7654, tensport = 6006)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        app_dir = options["app_dir"]
-        cmd = options["cmd"]
-        volume_from = options["volume"]
-
-        image_settings = {
-            'Image' => "#{image_name}:latest",
-
-            'Cmd' => cmd,
-            'WorkingDir' => app_dir,
-            'ExposedPorts' => {
-                '8888/tcp' => {},
-            },
-            'HostConfig' => {
-                'Binds' => ["/var/run/docker.sock:/var/run/docker.sock", "/usr/bin/docker:/usr/bin/docker"],
-                'PortBindings' => {
-                    '8888/tcp' => [
-                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
-                    ],
-                    '6006/tcp' => [
-                        {'HostPort' => "#{tensport}", 'HostIp' => 'localhost'}
-                    ],
-                },
-            },
-        }
-        container = Docker::Container.create(image_settings)
-        container.start()
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
-        # container.exec(command, tty: true)
-        # command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
-        # container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        say "#{container.id}:#{port}##{tensport}"
-      rescue => e
-        puts e
-        if e.message.include? "is not running"
-          return config_remote(image_name, port - 1, tensport - 1)
-        end
-
-        if container
-          container.kill()
-        end
-        return false
-      end
-    end
-
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-
-    def config_netrc(container)
-
-      login_content = options["login"]
-
-      container = Docker::Container.get(container)
-      command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-      container.exec(command, tty: true)
-      command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-      container.exec(command, tty: true)
-      command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-      container.exec(command, tty: true)
-      say "OK"
-
-    end
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
-    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
-    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
-
-
-    def config_remote_gpu(image_name, port = 7654, tensport = 6006)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        app_dir = options["app_dir"]
-        cmd = options["cmd"]
-
-        # image_settings = {
-        #     'Image' => "#{image_name}:latest",
-        #     'User' => 'ds',
-        #     'Cmd' => cmd,
-        #     'WorkingDir' => app_dir,
-        #     'ExposedPorts' => {
-        #         '8888/tcp' => {},
-        #     },
-        #     'HostConfig' => {
-        #         'PortBindings' => {
-        #             '8888/tcp' => [
-        #                 {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
-        #             ],
-        #             '6006/tcp' => [
-        #                 {'HostPort' => "6006", 'HostIp' => 'localhost'}
-        #             ],
-        #         },
-        #     },
-        # }
-
-        container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi  -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
-        container_id = container_id.gsub("\n", "")
-        container = Docker::Container.get(container_id)
-        # container.start()
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        say "#{container.id}:#{port}##{tensport}"
-      rescue => e
-        if e.message.include? "is not running"
-          puts "running asgain with: #{port - 1} #{tensport - 1}"
-          return config_remote_gpu(image_name, port - 1, tensport - 1)
-        end
-
-        if container
-          container.kill()
+    desc 'Collect and send job utilization', '', :hide => true
+    method_option :docker_id, :type => :string, :aliases => ["--docker_id"], :desc => "docker id to watch"
+    method_option :is_on_gpu, :type => :boolean, :aliases => ["--is_on_gpu"], :desc => "is on gpu", :default => true
+    def get_utilization()
+      @exp = Experiment.new(ENV['CNVRG_OWNER'], ENV['CNVRG_PROJECT'], job_id: ENV['CNVRG_JOB_ID'])
+      docker_id = options["docker_id"]
+      while true do
+        sleep 30
+        begin
+          stats = usage_metrics_in_docker(docker_id)
+          if options["is_on_gpu"]
+            gu = gpu_util(take_from_docker: true, docker_id: docker_id)
+            stats['gpu_util'] = gu[0]
+            stats['gpu'] = gu[1]
+          end
+          stats['docker_id'] = docker_id
+          @exp.send_machine_stats [stats] unless stats.empty?
+        rescue => e
+          log_error(e)
+          log_message("Failed to upload ongoing stats, continuing with experiment", Thor::Shell::Color::YELLOW)
         end
-        return false
       end
     end
 
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-
-    def config_flask_remote(image_name, port = 80)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        image_settings = {
-            'Image' => "#{image_name}:latest",
-            'User' => 'ds',
-            'Cmd' => '/usr/local/cnvrg/start_super.sh',
-            'WorkingDir' => '/home/ds/app',
-            'ExposedPorts' => {
-                '80/tcp' => {},
-            },
-            'HostConfig' => {
-                'PortBindings' => {
-                    '80/tcp' => [
-                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
-                    ],
-                },
-            },
-        }
-        container = Docker::Container.create(image_settings)
-        container.start()
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        say "#{container.id}:#{port}"
-      rescue => e
-        pus e
-        if e.message.include? "is not running"
-          return "port is taken"
-        end
-        puts "error"
-        if container
-          container.kill()
-        end
-        return false
-      end
-    end
-
-    desc '', '', :hide => true
-    method_option :login, :type => :string, :aliases => ["-l"], :default => ""
-
-    def config_flask_remote_gpu(image_name, port = 80)
-      local_images = Docker::Image.all
-
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.eql? "#{image_name}:latest"}.flatten
-      if docker_image_local.empty?
-        say "no image"
-        exit(1)
-      end
-
-      begin
-        login_content = options["login"]
-        container_id = `nvidia-docker run -itd -p 80:80 -w /home/ds/app #{image_name}:latest /usr/local/cnvrg/start_super.sh`
-        container_id = container_id.gsub("\n", "")
-        container = Docker::Container.get(container_id)
-        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
-        container.exec(command, tty: true)
-        say "#{container.id}:#{port}"
-      rescue => e
-        puts e
-        if e.message.include? "is not running"
-          return "port is taken"
-        end
-        puts "error"
-        if container
-          container.kill()
-        end
-        return false
-      end
-    end
 
     desc '', '', :hide => true
 
@@ -4933,39 +4584,10 @@ module Cnvrg
 
     end
 
-    desc 'upload_image', 'Upload new docker image to cnvrg', :hide => true
-    method_option :workdir, :type => :string, :aliases => ["-w","--workdir"], :desc => "workdir of docker image", :default => "/root"
-    method_option :description, :type => :string, :aliases => ["-d", "--description"], :desc => "description for docker image", :default => ""
-    method_option :user, :type => :string, :aliases => ["-u","--user"], :default => "root"
-    method_option :gpu, :type => :boolean, :aliases => ["-g","--gpu"], :default => false
-    def upload_image(image_name,image_path)
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-
-        @image = Cnvrg::Images.new()
-        say "Uploading new docker image file", Thor::Shell::Color::BLUE
-        workdir = options[:workdir]
-        description = options[:description]
-        user = options[:user]
-        is_gpu = options[:gpu]
-        res = @image.upload_docker_image(image_path, image_name, workdir, user, description, is_gpu)
-        if res["status"] == 200
-          image_slug = res["id"]
-          owner = CLI.get_owner
-          image_url = "#{Cnvrg::Helpers.remote_url}/#{owner}/settings/images/#{image_slug}"
-          log_message("Successfully uploaded image: #{image_url}", Thor::Shell::Color::GREEN, true)
-
-
-        else
-          log_message("Couldn't upload image: #{image_name}", Thor::Shell::Color::RED, true)
-
-        end
-      rescue => e
-        log_error(e)
-      end
-
-
+    desc 'file_exists', description: '', hide: true
+    def file_exists(file)
+      exit(0) if File.exists? file
+      exit(1)
     end
 
 
@@ -5145,29 +4767,40 @@ module Cnvrg
     method_option :project_slug, :type => :string, :aliases => ["-s"], :desc => "project slug"
     method_option :project_owner, :type => :string, :aliases => ["-o"], :desc => "project slug"
     method_option :frequency, :type => :numeric, :aliases => ["-f"], :desc => "poll frequency"
+    method_option :fetch_slugs, :type => :boolean, :default => false, :desc => "Fetch experiments slugs to compare"
 
     def compare_experiments
       verify_logged_in(true)
       log_start(__method__, args, options)
       exps_map = {}
+      copied_commits = []
 
-      if options[:slugs].blank?
+      if options[:slugs].blank? and options[:fetch_slugs].blank?
         log_message("No experiments slugs given", Thor::Shell::Color::RED)
         return false
       end
-      slugs = options[:slugs].split(",")
-      if slugs.blank?
-        log_message("No experiments slugs given", Thor::Shell::Color::RED)
-        return false
+      if options[:slugs].present?
+        slugs = options[:slugs].split(",")
       end
+
       frequency = options[:frequency] || 5
       namespace = options[:namespace]
       project_dir = is_cnvrg_dir(Dir.pwd)
       @project = Project.new(project_home=project_dir, slug: options[:project_slug], owner: options[:project_owner])
+      fetch_slugs = options[:fetch_slugs]
+      webapp_slug = ENV["CNVRG_JOB_ID"]
+      if fetch_slugs and webapp_slug.present?
+        slugs = @project.fetch_webapp_slugs(webapp_slug)
+      end
+      if slugs.blank?
+        log_message("No experiments slugs given", Thor::Shell::Color::RED)
+        return false
+      end
 
+      log_message("compare is running")
       while true
+        log_message("compare is running for slugs #{slugs}")
         slugs.each do |exp_slug|
-
           begin
             if exps_map[exp_slug].blank?
               exp = @project.get_experiment(exp_slug)["experiment"]
@@ -5181,15 +4814,23 @@ module Cnvrg
               log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
               Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
               exps_map[exp_slug] = exp
-            elsif exp["machine_activity"].present?
+            else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
-              Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+              success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+              if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
+                log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
+                Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
+                copied_commits << exp["last_successful_commit"]
+              end
             end
           rescue => e
             Cnvrg::Logger.log_error(e)
           end
         end
         sleep frequency
+        if fetch_slugs
+          slugs = @project.fetch_webapp_slugs(webapp_slug, slugs: slugs)
+        end
       end
     end
 
@@ -5275,127 +4916,6 @@ module Cnvrg
     end
 
 
-    desc 'pull_image', 'downloads and loads an image', :hide => true
-
-    def pull_image(image_name)
-      begin
-        verify_logged_in(false)
-        log_start(__method__, args, options)
-        owner = Cnvrg::CLI.get_owner()
-        image = Cnvrg::Images.image_exist(owner, image_name)
-        if !image
-          log_message("Couldn't find image in cnvrg repository", Thor::Shell::Color::RED)
-          exit(1)
-        end
-        path = download_image(image_name, image["slug"])
-        if path
-          log_message("Building image", Thor::Shell::Color::BLUE)
-          Docker.options[:read_timeout] = 216000
-          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
-            begin
-              if (log = JSON.parse(v)) && log.has_key?("stream")
-                next if log["stream"].starts_with? "Step"
-                $stdout.puts log["stream"]
-              end
-            rescue
-            end
-
-          end
-
-          if not image.nil?
-            FileUtils.rm_rf(path)
-            checks = Helpers.checkmark()
-            log_message("#{checks} Image built successfully", Thor::Shell::Color::GREEN)
-            return image
-          else
-
-            log_message("Could not build image", Thor::Shell::Color::RED)
-            return false
-          end
-        else
-
-          log_message("Could not download image", Thor::Shell::Color::RED)
-          return false
-
-
-        end
-
-          # else
-          #   path = download_image(image_name,image["slug"])
-          #   if path
-          #     image = Docker::Image.import(path)
-          #     image.tag('repo' => image_name, 'tag' => 'latest')
-          #     if not image.nil?
-          #       say "Finished downloading image, cleaning up..", Thor::Shell::Color::GREEN
-          #       FileUtils.rm(path)
-          #       checks = Helpers.checkmark()
-          #       say "#{checks} Done", Thor::Shell::Color::GREEN
-          #       log_end(0)
-          #       return image
-          #       log_end(0)
-          #     else
-          #       say "Could not download image", Thor::Shell::Color::RED
-          #       return false
-          #     end
-          #
-          #   end
-          # end
-      rescue => e
-
-        log_message "Error: couldn't build image", Thor::Shell::Color::RED
-        log_error(e)
-
-      rescue SignalException
-        say "\nAborting"
-        exit(1)
-      ensure
-        if path
-          FileUtils.rm_rf(path)
-
-        end
-      end
-
-
-    end
-
-    desc 'set_image', 'set image to a porject', :hide => true
-
-    def set_image(docker_image)
-      verify_logged_in(true)
-      log_start(__method__, args, options)
-      working_dir = is_cnvrg_dir
-      project = Project.new(working_dir)
-
-      local_images = Docker::Image.all
-      docker_image_local = local_images.map {|x| x.info["RepoTags"]}.flatten.select {|y| y.include? docker_image}.flatten
-      if docker_image_local.size == 0
-
-        if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
-          image = pull(docker_image)
-          if image
-            log_message("downloaded image: #{docker_image}", Thor::Shell::Color::BLUE)
-            @image = Images.new(working_dir, docker_image)
-          else
-            log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-            exit(1)
-          end
-        else
-          log_message("Could not create a new project with docker, image was not found", Thor::Shell::Color::RED)
-          exit(1)
-
-        end
-      elsif docker_image_local.size == 1
-        log_message("found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE)
-        @image = Images.new(working_dir, docker_image_local[0])
-      elsif docker_image_local.size > 1
-        log_message("found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE)
-        image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
-        image_name = image_name.strip
-        @image = Images.new(working_dir, image_name)
-      end
-      @image.update_image_activity(project.last_local_commit, nil)
-    end
-
     desc 'check_pod_restart', 'Check pod restart', :hide => true
     def check_pod_restart
       Cnvrg::CLI.new.log_start(__method__, args, options)
@@ -5670,7 +5190,7 @@ module Cnvrg
 
         if dirs.size == 0
           log_message("Couldn't find cnvrg directory. Please start a new project", Thor::Shell::Color::RED)
-
+          puts Thread.current.backtrace
           exit(1)
         end
         return dirs.join("/")
@@ -5773,7 +5293,7 @@ module Cnvrg
           is_cnvrg = is_cnvrg_dir
           if !is_cnvrg
             say "You're not in a cnvrg project directory", Thor::Shell::Color::RED
-            exit(0)
+            exit(1)
           end
 
         end
@@ -5919,21 +5439,6 @@ module Cnvrg
 
       end
 
-      def container_changes(dir)
-        container_id = is_project_with_docker(dir)
-        if not container_id
-          return false
-        end
-        container = Docker::Container.get(container_id)
-        command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
-        pip = container.exec(command, tty: true)[0]
-        command = ["/bin/bash", "-lc", "dpkg -l"]
-        dpkg = container.exec(command, tty: true)[0]
-        command = ["/bin/bash", "-lc", "cat /home/ds/.bash_history"]
-        history = container.exec(command, tty: true)[0]
-        diff = [pip, dpkg, history]
-        return diff
-      end
 
       def is_port_taken(ip = Cnvrg::CLI::IP, port = Cnvrg::CLI::PORT, seconds = 1)
         Timeout::timeout(seconds) do
@@ -6116,13 +5621,17 @@ module Cnvrg
 
       end
 
-      def gpu_util
+      def gpu_util(take_from_docker: false, docker_id: nil)
         if !Helpers.ubuntu?
           return 0.0
         end
         stats = [[],[]]
         begin
-          gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv`
+          if take_from_docker
+            gpu_stats = `docker exec -it #{docker_id} sh -c 'nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv'`
+          else
+            gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv`
+          end
 
           if !gpu_stats.nil?
             gpu_stats = gpu_stats.split("\n")[1..-1]