cnvrg 0.0.11 → 0.0.14

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 745fd9b806d9e5d46b4ba24212e06f5a426447f3
-  data.tar.gz: c950e20258da5c1243765572b6b7f4c93d0cad1a
+  metadata.gz: 27f1b54ab10e9cc75d4c290fa0802c11a148275c
+  data.tar.gz: 4e80f1fbb3b2d3c4bd5450708359de08212105a3
 SHA512:
-  metadata.gz: f60cd3a8cbb4d9bb6c53beac4faf0b76d5db9db469e4474dabe36787985b7d601a07d0029dcb797b7ede238ec2fa3e96beaaa56497b39fc05380dd05f54981fb
-  data.tar.gz: 6587fe66cf021518dc9249a3c0eb936ccfb126b51b6b6cc605d1e44b09cbee30b55bb590cc7420bec05951077d63f452bb9a88a7466a78bd1281d51ccd334b1f
+  metadata.gz: eb9766dc5c54ad8b5d8e4463178dc0cd696e76c214dd1e8d89d67f23369d68e716647ac253484c65f273352196530626b9e1a35f932b4484226f603c12056079
+  data.tar.gz: 009acaba20cd093a9ed713ba96025b65f981616a47d22e63ec6474d4d91677afef99f141b976c78727f193f5524b5587569d1bf6c4948bb9a0cab6866767c81c

data/cnvrg.gemspec

@@ -35,7 +35,11 @@ Gem::Specification.new do |spec|
     spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
     spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
     spec.add_runtime_dependency 'logstash-logger'
+    spec.add_runtime_dependency 'launchy'
     spec.add_runtime_dependency 'docker-api'
+    spec.add_runtime_dependency 'rubyzip'
+    spec.add_runtime_dependency 'activesupport'
+
     
 end
 

data/lib/cnvrg.rb

@@ -1,8 +1,7 @@
 require 'rubygems'
-require 'thor'
-
 require 'cnvrg/version'
 require 'cnvrg/cli'
+require 'thor'
 
 
 module Cnvrg

data/lib/cnvrg/Images.rb

@@ -0,0 +1,347 @@
+require 'fileutils'
+require 'cnvrg/files'
+require 'docker'
+
+module Cnvrg
+  class Images
+    attr_reader :image_name, :image_tag, :is_docker, :project_slug, :commit_id, :owner, :port, :image_slug
+
+
+    def initialize(working_dir, image_name="")
+      begin
+        config = YAML.load_file(working_dir+"/.cnvrg/config.yml")
+        idx = YAML.load_file(working_dir + "/.cnvrg/idx.yml")
+
+        @working_dir = working_dir
+        @commit_id =idx[:commit]
+        @project_title = config[:project_name]
+        @project_slug = config[:project_slug]
+        @owner = config[:owner]
+        @is_docker = config[:docker]
+        if image_name.empty?
+          @image_name = config[:image_base]
+          @image_tag = config[:image_tag]
+          @image_slug = find_image()
+
+        else
+
+          @image_name = image_name
+          if !@image_name.nil? and !@image_name.empty?
+            if image_name.include? ":"
+            @image_name = image_name[0, image_name.index(":")]
+            @image_tag = image_name[image_name.index(":")+1, image_name.size]
+            else
+              @image_name = image_name
+              @image_tag = "lastest"
+            end
+
+          end
+          @image_slug = find_image(false)
+          update_image_activity(@commit_id, nil)
+          config = {project_name: config[:project_name],
+                    project_slug: config[:project_slug],
+                    owner: config[:owner],
+                    docker: true, image_base: @image_name, image_tag: @image_tag, image_slug: image_slug}
+          File.open(working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        end
+
+      rescue => e
+        puts e
+      end
+
+    end
+
+    def is_container_exist()
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      if config[:container].nil?
+        return false
+      end
+      return config[:container]
+    end
+
+    def container_port()
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      if config[:container].nil?
+        return false
+      else
+        return config[:port]
+      end
+    end
+
+    def self.image_exist(owner, image_name)
+
+      image_res = Cnvrg::API.request("users/#{owner}/images/" + "find", 'POST', {image_name: image_name})
+
+      if Cnvrg::CLI.is_response_success(image_res)
+        image= image_res["result"]["image"]
+        return image
+      else
+        return false
+
+      end
+    end
+
+    def create_custom_image(new_image_name)
+      new_image_name = @project_slug+"_"+@commit_id unless !new_image_name.empty?
+      python_arr = get_installed_packages("python")
+      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      system_arr = get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      bash_history = get_bash_history
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/custom", 'POST', {image_slug: @image_slug, libraries: py,
+                                                                                             dpkg: sys, new_image_name: new_image_name})
+      if Cnvrg::CLI.is_response_success(response) and !response["result"]["slug"].nil?
+        container = get_container()
+        update_image(new_image_name+":latest", container, response["result"]["slug"])
+      end
+
+      return response
+
+    end
+
+    def update_image(image_name, container, image_slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      @image_name = image_name
+      if !@image_name.nil? and !@image_name.empty?
+        @image_name = image_name[0, image_name.index(":")]
+        @image_tag = image_name[image_name.index(":")+1, image_name.size]
+      end
+      config = {project_name: config[:project_name],
+                project_slug: config[:project_slug],
+                owner: config[:owner],
+                docker: true, image_base: @image_name, image_tag: @image_tag, container: container.id, image_slug: image_slug}
+
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+
+    def get_container(stop=false)
+      begin
+        container_id=is_container_exist()
+
+        if !container_id
+          return create_container()
+        else
+          container = Docker::Container.get(container_id)
+          status = container.json["State"]["Status"]
+
+          if status == "running"
+            return container
+          else
+            if stop
+              return false
+            end
+            res = container.start()
+            if res.info["State"]["Status"].eql? "exited" and  res.info["State"]["Error"].include? "port is already allocated"
+              return create_container()
+            end
+            return container
+          end
+        end
+      rescue => e
+        if e.message.include? "No such container"
+
+          return create_container()
+        else
+          return false
+        end
+      end
+
+    end
+
+    def create_container(port=7654, is_remote=false)
+      begin
+        image_settings = {
+            'Image' => "#{@image_name}:#{@image_tag}",
+            'User' => 'ds',
+            'Cmd' => '/home/ds/run_ipython.sh',
+            'ExposedPorts' => {
+                '8888/tcp' => {},
+            },
+            'HostConfig' => {
+                'Binds' => ["#{@working_dir}:/home/ds/notebooks"],
+                'PortBindings' => {
+                    '8888/tcp' => [
+                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
+                    ],
+                },
+            },
+        }
+        # if !is_remote
+        #   image_settings['HostConfig'].merge!({ 'Binds' => ["#{@working_dir}:/home/ds/notebooks"]})
+        # end
+        container = Docker::Container.create(image_settings)
+        container.start()
+        netrc = File.open(File.expand_path('~')+"/.netrc", "rb")
+        netrc_content = netrc.read
+        container.store_file("/home/ds/.netrc", netrc_content)
+        command = ["/bin/bash", "-lc", "sudo chmod 600 /home/ds/.netrc"]
+        p = container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.netrc"]
+        p = container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        container.exec(command, tty: true)
+        config = File.open(File.expand_path('~')+"/.cnvrg/config.yml", "rb")
+        config_content = config.read
+        container.store_file("/home/ds/.cnvrg/config.yml", config_content)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        # Libraries instlled
+        save_installed_libraries(container)
+        config = {project_name: @project_name,
+                  project_slug: @project_slug,
+                  owner: @owner,
+                  docker: true, image_base: @image_name, image_tag: @image_tag, container: container.id, port: port, image_slug: @image_slug}
+
+        File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+
+        return container
+
+
+      rescue => e
+        if e.message.include? "is not running"
+          return create_container(port-1)
+        end
+        puts e
+        return false
+      rescue SignalException
+
+        say "\nAborting", Thor::Shell::Color::RED
+        exit(1)
+      end
+
+
+    end
+
+    def save_installed_libraries(container)
+      begin
+        command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
+        pip = container.exec(command, tty: true)[0]
+        command = ["/bin/bash", "-lc", "dpkg -l | grep '^ii' | awk '{print $2\"==\"$3}'"]
+        dpkg = container.exec(command, tty: true)[0]
+        File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
+        File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
+      rescue => e
+        puts e
+      end
+
+
+    end
+
+    def remote_notebook(notebook_path, instance_type, kernel)
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/remote_notebook", 'POST', {dir: notebook_path,
+                                                                                                      project_slug: @project_slug,
+                                                                                                      instance_type: instance_type,
+                                                                                                      kernel: kernel})
+      return response
+    end
+
+    def get_installed_packages(repo)
+      container = get_container()
+      case repo
+        when "python"
+          command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
+        when "system"
+          command = ["/bin/bash", "-lc", "dpkg -l | grep '^ii' | awk '{print $2\"==\"$3}'"]
+      end
+
+      libs = container.exec(command, tty: true)[0]
+      libs_arr = libs.join("").split("\r\n")
+      return libs_arr
+
+    end
+
+    def get_bash_history
+      container = get_container()
+      command = ["/bin/bash", "-lc", "cat /home/ds/.bash_history"]
+      history = container.exec(command, tty: true)[0][0]
+      if history.include? "No such file"
+        history = ""
+      end
+      return history
+    end
+
+
+    def get_image_state
+      python_arr = self.get_installed_packages("python")
+      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      system_arr = self.get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      bash_history = self.get_bash_history
+      diff = [py, sys, bash_history]
+
+    end
+
+    def find_image(update=true)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      image_slug = config[:image_slug]
+      if image_slug.nil? or image_slug.empty?
+        image_res = Cnvrg::API.request("users/#{@owner}/images/" + "find", 'POST', {image_name: @image_name})
+
+        if Cnvrg::CLI.is_response_success(image_res)
+          image_slug = image_res["result"]["image"]["slug"]
+          update_slug(image_slug) unless !update
+          return image_slug
+        end
+      else
+        return image_slug
+
+      end
+    end
+
+    def set_note_url(note_slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:notebook_slug] = note_slug
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+    def note_slug
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      note_slug = config[:notebook_slug]
+      if note_slug.nil? or note_slug.empty?
+        return false
+      else
+        return note_slug
+      end
+    end
+    def remove_note_slug
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:notebook_slug] = ""
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+
+    end
+
+
+    def update_slug(slug)
+      config = YAML.load_file(@working_dir+"/.cnvrg/config.yml")
+      config[:image_slug] = slug
+      File.open(@working_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+    end
+
+    def new_machine(instance_type)
+      title = "#{instance_type} auto created by cli"
+      response = Cnvrg::API.request("users/#{@owner}/machines/new", 'POST', {machine_name: title, instance_type: instance_type})
+      return Cnvrg::CLI.is_response_success(response)
+
+    end
+
+    def update_image_activity(commit, exp_slug)
+      response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/update_activity", 'POST', {commit: commit, project_slug: @project_slug, experiment: exp_slug})
+      return Cnvrg::CLI.is_response_success(response)
+    end
+
+    def handle_image_activity
+      home_dir = File.expand_path('~')
+      zip_dir = "#{home_dir}/.cnvrg/tmp/config.zip"
+      compress = `zip -j #{zip_dir} #{home_dir}/.netrc #{home_dir}/.cnvrg/config.yml`
+      @files = Cnvrg::Files.new(@owner, @project_slug)
+      res_id = @files.upload_exec_file(zip_dir, @image_name, @commit_id)
+      FileUtils.remove zip_dir
+      return res_id
+    end
+
+
+  end
+
+end

data/lib/cnvrg/api.rb

@@ -80,10 +80,12 @@ module Cnvrg
                 # what if windows?
                 # data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
                 file_base = File.basename(data[:relative_path])
-                temp_path = File.expand_path('~')+"/.cnvrg/tmp/#{file_base}"
+                temp_path = File.expand_path('~')+"/.cnvrg/tmp_files/#{file_base}"
                 FileUtils.touch(temp_path)
                 data[:file] = Faraday::UploadIO.new("#{temp_path}", "plain/text")
+
                 response = conn.post "#{endpoint_uri}/#{resource}", data
+
                 FileUtils.rm(temp_path)
 
                 if parse_request == true
@@ -101,7 +103,8 @@ module Cnvrg
                 end
             else
             end
-            rescue
+            rescue => e
+                puts e
                return nil
             end
 

data/lib/cnvrg/cli.rb

@@ -17,10 +17,18 @@ require 'cnvrg/auth'
 require 'cnvrg/project'
 require 'cnvrg/files'
 require 'cnvrg/experiment'
+require 'cnvrg/Images'
 require 'etc'
 require 'logstash-logger'
 require 'cnvrg/job'
 require 'docker'
+require 'launchy'
+require 'socket'
+require 'timeout'
+require 'fileutils'
+require 'zip'
+require 'active_support/all'
+require 'thor'
 
 # DEV VERSION
 #
@@ -28,23 +36,62 @@ module Cnvrg
   class CLI < Thor
 
     INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
-    desc 'test', 'Prints cnvrg current version'
+    IP="localhost"
+    PORT=7654
+    desc '', ''
+
+    def printable_commands(all = true, subcommand = false)
+      (all ? all_commands : commands).map do |_, command|
+        next if command.hidden? or (command.description.empty? and command.usage.empty?)
+        item = []
+        item << banner(command, false, subcommand)
+        item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
+        item
+      end.compact
+    end
+
+    class << self
+      # Hackery.Take the run method away from Thor so that we can redefine it.
+      def is_thor_reserved_word?(word, type)
+        return false if word == "run"
+        super
+      end
+    end
+
+    desc "", ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
 
     def test
-      verify_software_installed("docker")
-      container =  Docker::Container.create( 'Image' => 'cnvrgio/python2')
-      # container =  Docker::Container.create("1e4e23560813")
-      container.start
-      puts container.exec(['bash'], stdin: StringIO.new("python --version"))
+      # image_settings = {
+      #     'Image' => "cnvrg:latest",
+      #     'User' => 'ds',
+      #     'Cmd' => '/home/ds/run_ipython.sh',
+      #     'ExposedPorts' => {
+      #         '80/tcp' => {},
+      #     },
+      #     'HostConfig' => {
+      #         'PortBindings' => {
+      #             '80/tcp' => [
+      #                 {'HostPort' => "7654", 'HostIp' => 'localhost'}
+      #             ],
+      #         },
+      #     },
+      # }
+      container = Docker::Container.get('b4d64bf83f41')
+      s = "/leah/1/2/3/4/5"
+      command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+      puts container.exec(command, tty: true)
 
     end
 
+
     desc 'version', 'Prints cnvrg current version'
 
     def version
       puts Cnvrg::VERSION
 
     end
+
     map %w(-v --version) => :version
 
     desc 'set api url', 'set api url'
@@ -62,22 +109,56 @@ module Cnvrg
           FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
         end
         config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        owner = config.to_h[:owner]
+
         say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
         if config.empty?
           config = {owner: "", username: "", version_last_check: get_start_day(), api: url}
         else
           config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url}
         end
-        checks = Helpers.checkmark
+        res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
+        if Cnvrg::CLI.is_response_success(res)
 
+          checks = Helpers.checkmark
+
+
+          File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          say "#{checks} Done", Thor::Shell::Color::GREEN
+        else
+          say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
+          exit(1)
+
+        end
 
-        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-        say "#{checks} Done", Thor::Shell::Color::GREEN
       rescue
         say "Couldn't set default api, contact help@cnvrg.io", Thor::Shell::Color::RED
       end
     end
 
+    desc '', ''
+
+    def set_remote_api_url(owner, current_user, url)
+      home_dir = File.expand_path('~')
+      if !url.end_with? "/api"
+        url = url+"/api"
+      end
+      begin
+        if !File.directory? home_dir+"/.cnvrg"
+          FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
+        end
+        if !File.exist?(home_dir+"/.cnvrg/config.yml")
+          FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
+        end
+        config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+        config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url}
+        File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        say "Done"
+      rescue
+        say "ERROR", Thor::Shell::Color::RED
+      end
+    end
+
     map %w(-api --api) => :set_api_url
 
     desc 'set default owner', 'set default owner'
@@ -97,11 +178,29 @@ module Cnvrg
       if Cnvrg::CLI.is_response_success(res)
         owner = username
         result = res["result"]
-        if result["owners"].size > 1
-          owner = ask("Choose default owner:\n"+result["owners"].join("\n")+"\n")
+        owners = result["owners"]
+        urls = result["urls"]
+        choose_owner = result["username"]
+
+        if owners.empty?
+        else
+          owners << choose_owner
+          chosen = false
+          while !chosen
+            choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
+
+            owners_lower = owners.map { |o| o.downcase }
+            ow_index = owners_lower.index(choose_owner.downcase)
+            if ow_index.nil?
+              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+            else
+              chosen = true
+            end
+          end
+
 
         end
-        if set_owner(owner, username)
+        if set_owner(owner, username,urls[ow_index])
           say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
         else
           say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
@@ -138,6 +237,7 @@ module Cnvrg
 
           say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
           owners = result["owners"]
+          urls = result["urls"]
           choose_owner = result["username"]
 
           if owners.empty?
@@ -145,18 +245,21 @@ module Cnvrg
             owners << choose_owner
             chosen = false
             while !chosen
-            choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
-            owners_lower = owners.map{|o| o.downcase}
-            if !owners_lower.include? choose_owner.downcase
-              say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
-            else
-              chosen = true
-            end
+              choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
+
+              owners_lower = owners.map { |o| o.downcase }
+              ow_index = owners_lower.index(choose_owner.downcase)
+              if ow_index.nil?
+                say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
+              else
+                chosen = true
+              end
             end
 
 
           end
-          if set_owner(choose_owner, result["username"])
+
+          if set_owner(choose_owner, result["username"], urls[ow_index])
             say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
 
           else
@@ -172,7 +275,7 @@ module Cnvrg
         end
       rescue SignalException
 
-        say "/nAborting"
+        say "\nAborting"
         logout()
         exit(1)
       end
@@ -188,7 +291,7 @@ module Cnvrg
         netrc.save
         say "Logged out successfully.\n", Thor::Shell::Color::GREEN
       rescue SignalException
-        say "/nAborting"
+        say "\nAborting"
         exit(1)
       end
 
@@ -201,7 +304,7 @@ module Cnvrg
       begin
 
         verify_logged_in(false)
-        log_start(__method__,args,options)
+        log_start(__method__, args, options)
         auth = Cnvrg::Auth.new
         if (email = auth.get_email)
           say "Logged in as: #{email}", Thor::Shell::Color::GREEN
@@ -213,13 +316,12 @@ module Cnvrg
       rescue SignalException
         log_end(-1)
 
-        say "/nAborting"
+        say "\nAborting"
         exit(1)
       end
     end
 
     ## Projects
-
     desc 'new', 'Create a new cnvrg project'
     method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
     method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
@@ -227,142 +329,296 @@ module Cnvrg
     def new(project_name)
       begin
         verify_logged_in(false)
-        log_start(__method__,args,options)
+        log_start(__method__, args, options)
         clean = options["clean"]
         docker_image = options["docker_image"]
+        working_dir = Dir.pwd + "/" + project_name
+        docker = false
         if !docker_image.nil? and !docker_image.empty?
-          # local_images = Docker::Image.all
-          # docker_image_local = local_images.map{|x| x.info["RepoTags"]}.select{|y| y[0].include? docker_image}.flatten
-          # if docker_image_local.size == 0
-          #   if yes? "Image wasn't found locally, pull image from cnvrg repository?"
-          #     image = pull_image(docker_image)
-          #     if image
-          #       say "downloaded image: #{docker_image_local[0]}, creating a container.."
-          #       @container = Docker::Container.create('Image' => '#{docker_image_local[0]}')
-          #       if @container.nil?
-          #         say "could not create container with image #{docker_image_local[0]}"
-          #       end
-          #     end
-          #   else
-          #     say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
-          #     exit(1)
-          #
-          #   end
-          # elsif docker_image_local.size == 1
-          #   say "found image: #{docker_image_local[0]}, creating a container.."
-          #   @container = Docker::Container.create('Image' => '#{docker_image_local[0]}')
-          #   if @container.nil?
-          #     say "could not create container with image #{docker_image_local[0]}"
-          #   end
-          #
-          # end
+          docker = true
         end
         say "Creating #{project_name}", Thor::Shell::Color::BLUE
         if Dir.exists? project_name or File.exists? project_name
           say "Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
-          log_end(1,"conflict with dir/file #{project_name}")
+          log_end(1, "conflict with dir/file #{project_name}")
           exit(1)
-        end
 
-        if Project.create(project_name, clean)
+        end
+        if Project.create(project_name, clean, with_docker=docker)
           path = Dir.pwd + "/" + project_name
           @project = Project.new(path)
           @project.generate_idx
+          if docker
+            local_images = Docker::Image.all
+            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.include? docker_image }.flatten
+            if docker_image_local.size == 0
+
+              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+                image = pull_image(docker_image)
+                if image
+                  say "downloaded image: #{docker_image}"
+                  @image = Images.new(working_dir, docker_image)
+                else
+                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  @project.revert(working_dir)
+                  exit(1)
+                end
+              else
+                say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                @project.revert(working_dir)
+                exit(1)
+
+              end
+            elsif docker_image_local.size == 1
+              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, docker_image_local[0])
+            elsif docker_image_local.size >1
+              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+              image_name = image_name.strip
+              @image = Images.new(working_dir, image_name)
+            end
+            @image.update_image_activity(nil, nil)
+
+          end
         else
           say "Error creating project, please contact support.", Thor::Shell::Color::RED
-          log_end(1,"can't create project #{project_name}")
+          @project.revert(working_dir)
+          log_end(1, "can't create project #{project_name}")
 
           exit(0)
         end
 
-        say "created\t\tproject's tree", Thor::Shell::Color::GREEN
-        say "created\t\tproject's config", Thor::Shell::Color::GREEN
+        say "created project successfully", Thor::Shell::Color::GREEN
         say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
+      rescue => e
+        puts e
+        log_end(-1)
+        if Dir.exist? working_dir
+
+          @project.revert(working_dir)
+        end
+
+        exit(1)
 
       rescue SignalException
         log_end(-1)
+        if Dir.exist? working_dir
 
-        say "/nAborting"
+          @project.revert(working_dir)
+        end
+        say "\nAborting"
         exit(1)
       end
     end
+    desc 'set_image', 'set_image for a project'
+
+    def set_image(docker_image)
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir
+      local_images = Docker::Image.all
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.include? docker_image }.flatten
+      if docker_image_local.size == 0
+
+        if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+          image = pull_image(docker_image)
+          if image
+            say "downloaded image: #{docker_image}"
+            @image = Images.new(working_dir, docker_image)
+          else
+            say "Could not set image, image was not found", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Could nset image, image was not found", Thor::Shell::Color::RED
+          exit(1)
 
+        end
+      elsif docker_image_local.size == 1
+        say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+        @image = Images.new(working_dir, docker_image_local[0])
+      elsif docker_image_local.size >1
+        say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+        image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+        image_name = image_name.strip
+        @image = Images.new(working_dir, image_name)
+      end
+      @image.update_image_activity(nil, nil)
 
+    end
+    desc 'link', 'Link current directory to a cnvrg project'
+    method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
+    method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
 
-      desc 'link', 'Link current directory to a cnvrg project'
-      method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
+    def link
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        docker_image = options["docker_image"]
+        if !docker_image.nil? and !docker_image.empty?
+          docker = true
+        else
+          docker = false
+        end
 
-      def link
-        begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
+        sync = options["sync"]
+        project_name =File.basename(Dir.getwd)
+        say "Linking #{project_name}", Thor::Shell::Color::BLUE
+        if File.directory?(Dir.getwd+"/.cnvrg")
+          config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
+          say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
 
-          sync = options["sync"]
-          project_name =File.basename(Dir.getwd)
-          say "Linking #{project_name}", Thor::Shell::Color::BLUE
-          if File.directory?(Dir.getwd+"/.cnvrg")
-            config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
-            say "Directory is already linked to #{config[:project_slug]}", Thor::Shell::Color::RED
+          exit(0)
+        end
+        working_dir = Dir.getwd
+        owner = CLI.get_owner
+        if Project.link(owner,project_name, docker)
+          path = Dir.pwd
+          @project = Project.new(path)
+          @project.generate_idx()
+          if docker
+            local_images = Docker::Image.all
+            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.include? docker_image }.flatten
+            if docker_image_local.size == 0
+              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+                image = pull_image(docker_image)
+                if image
+                  say "downloaded image: #{docker_image}"
+                  @image = Images.new(working_dir, docker_image)
+                else
+                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  @project.revert(working_dir)
+                  exit(1)
+                end
+              else
+                say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                @project.revert(working_dir)
+                exit(1)
 
-            exit(0)
-          end
-          if Project.link(project_name)
-            path = Dir.pwd
-            @project = Project.new(path)
-            @project.generate_idx()
-            if sync
-              upload(true)
+              end
+            elsif docker_image_local.size == 1
+              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, docker_image_local[0])
+            elsif docker_image_local.size >1
+              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, image_name)
             end
+          end
+          if sync
+            say "Syncing project", Thor::Shell::Color::BLUE
+            upload(true)
+          end
 
-            url = @project.url
-            say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::BLUE
-            log_end(0)
+          url = @project.url
+          check = Helpers.checkmark
+          say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
+          say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
+          log_end(0)
+
+        else
+          log_end(1, "can't link project")
+          @project.revert(working_dir) unless @project.nil?
+          say "Error linking project, please contact support.", Thor::Shell::Color::RED
+          exit(0)
+        end
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting"
+        exit(1)
+      end
+    end
+    desc 'unlink','Unlink a porject'
+    def unlink
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir()
+      list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
+      FileUtils.rm_rf list_to_del
+    end
+
+    desc 'clone', 'Clone a project'
+    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
+    method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
+
+    def clone(project_url)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        url_parts = project_url.split("/")
+        project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
+        slug = url_parts[project_index+1]
+        owner = url_parts[project_index-1]
+        remote = options["remote"] || false
+        response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
+        Cnvrg::CLI.is_response_success(response)
+        response = JSON.parse response["result"]
+        project_name = response["title"]
+        commit_to_clone = options["commit"] || nil
+
+        say "Cloning #{project_name}", Thor::Shell::Color::BLUE
+        clone_resp = false
+        if remote
+          clone_resp = Project.clone_dir_remote(slug, owner, project_name)
+          project_home = Dir.pwd
+        else
+          if (Dir.exists? project_name)
+            say "Error: Conflict with dir #{project_name}", Thor::Shell::Color::RED
+            if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
+              say "Remove dir in order to clone #{project_name}", Thor::Shell::Color::RED
+              log_end(1, "conflict with dir #{project_name}")
+
+              exit(1)
+            end
 
-          else
-            log_end(1,"can't link project")
-            say "Error linking project, please contact support.", Thor::Shell::Color::RED
-            exit(0)
           end
-        rescue SignalException
-          log_end(-1)
+          clone_resp = Project.clone_dir(slug, owner, project_name)
+          project_home = Dir.pwd+"/"+project_name
+
 
-          say "/nAborting"
-          exit(1)
         end
-      end
 
-      desc 'clone', 'Clone a project'
+        if clone_resp
+          @project = Project.new(project_home)
+          @files = Cnvrg::Files.new(@project.owner, slug)
+          response = @project.clone(remote, commit_to_clone)
+          Cnvrg::CLI.is_response_success response
+          working_dir = project_home
+          docker_image = response["result"]["image"]
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          if !docker_image.nil? and !docker_image.empty? and !remote
+            local_images = Docker::Image.all
+            docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
+            if docker_image_local.size == 0
+              if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+                image = pull_image(docker_image)
+                if image
+                  say "downloaded image: #{docker_image}"
+                  @image = Images.new(working_dir, docker_image)
+                else
+                  say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
+                  @project.revert(working_dir)
+                  exit(1)
+                end
+              end
+
+            elsif docker_image_local.size == 1
+              say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, docker_image_local[0])
+            elsif docker_image_local.size >1
+              say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+              image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+              @image = Images.new(working_dir, image_name)
+            end
 
-      def clone(project_url)
-        begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
-          url_parts = project_url.split("/")
-          project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
-          slug = url_parts[project_index+1]
-          owner = url_parts[project_index-1]
-          response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
-          Cnvrg::CLI.is_response_success(response)
-          response = JSON.parse response["result"]
-          project_name = response["title"]
-          say "Cloning #{project_name}", Thor::Shell::Color::BLUE
-          if Dir.exists? project_name or File.exists? project_name
-            log_end(1,"conflict with dir/file #{project_name}")
-
-            say "Error: Conflict with dir/file #{project_name}", Thor::Shell::Color::RED
-            exit(1)
           end
 
-          if Project.clone_dir(slug, owner, project_name)
-            project_home = Dir.pwd+"/"+project_name
-            @project = Project.new(project_home)
-            @files = Cnvrg::Files.new(@project.owner, slug)
-            response = @project.clone
-            Cnvrg::CLI.is_response_success response
-            idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
-            File.open(project_name + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
-            successful_changes = []
-            say "Downloading files", Thor::Shell::Color::BLUE
+          successful_changes = []
+          say "Downloading files", Thor::Shell::Color::BLUE
+          if !response["result"]["tree"].nil?
             response["result"]["tree"].each do |f|
               relative_path = f[0].gsub(/^#{@project.local_path}/, "")
               if f[0].end_with? "/"
@@ -372,202 +628,226 @@ module Cnvrg
                 end
               else
                 # blob
-                if @files.download_file(f[0], relative_path, project_home)
+                if @files.download_file_s3(f[0], relative_path, project_home)
                   successful_changes << relative_path
                 end
               end
             end
-            say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
-            log_end(0)
-          else
-            log_end(1,"can't create directory")
-
-            say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
-            exit(1)
           end
-        rescue SignalException
-          log_end(-1)
-          say "/nAborting"
+
+          say "Done.\nDownloaded total of #{successful_changes.size} files", Thor::Shell::Color::GREEN
+          log_end(0)
+        else
+          log_end(1, "can't create directory")
+
+          say "Error: Couldn't create directory: #{project_name}", Thor::Shell::Color::RED
           exit(1)
         end
-
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
       end
 
-      desc 'status', 'Show the working tree status'
-      def status
-        begin
-          verify_logged_in()
-          log_start(__method__,args,options)
-          @project = Project.new(get_project_home)
-          result = @project.compare_idx["result"]
-          commit = result["commit"]
-          result = result["tree"]
-          say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
-          if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
-            say "Project is up to date", Thor::Shell::Color::GREEN
-            log_end(0)
-            return true
-          end
-          if result["added"].size > 0
-            say "Added files:\n", Thor::Shell::Color::BLUE
-            result["added"].each do |a|
-              say "\t\tA:\t#{a}", Thor::Shell::Color::GREEN
-            end
+    end
+
+
+    desc 'status', 'Show the working tree status'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+
+
+    def status
+      begin
+        verify_logged_in()
+        log_start(__method__, args, options)
+        @project = Project.new(get_project_home)
+        new_branch = options["new_branch"] || false
+
+        result = @project.compare_idx(new_branch)["result"]
+        commit = result["commit"]
+        result = result["tree"]
+        say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
+        if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN
+          log_end(0)
+          return true
+        end
+        if result["added"].size > 0
+          say "Added files:\n", Thor::Shell::Color::BLUE
+          result["added"].each do |a|
+            say "\t\tA:\t#{a}", Thor::Shell::Color::GREEN
           end
+        end
 
-          if result["deleted"].size > 0
-            say "Deleted files:\n", Thor::Shell::Color::BLUE
-            result["deleted"].each do |a|
-              say "\t\tD:\t#{a}", Thor::Shell::Color::GREEN
-            end
+        if result["deleted"].size > 0
+          say "Deleted files:\n", Thor::Shell::Color::BLUE
+          result["deleted"].each do |a|
+            say "\t\tD:\t#{a}", Thor::Shell::Color::GREEN
           end
-          if result["updated_on_local"].size > 0
-            say "Local changes:\n", Thor::Shell::Color::BLUE
-            result["updated_on_local"].each do |a|
-              say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
-            end
+        end
+        if result["updated_on_local"].size > 0
+          say "Local changes:\n", Thor::Shell::Color::BLUE
+          result["updated_on_local"].each do |a|
+            say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
           end
+        end
 
-          if result["updated_on_server"].size > 0
-            say "Remote changes:\n", Thor::Shell::Color::BLUE
-            result["updated_on_server"].each do |a|
-              say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
-            end
+        if result["updated_on_server"].size > 0
+          say "Remote changes:\n", Thor::Shell::Color::BLUE
+          result["updated_on_server"].each do |a|
+            say "\t\tM:\t#{a}", Thor::Shell::Color::GREEN
           end
+        end
 
-          if result["conflicts"].size > 0
-            say "Conflicted changes:\n", Thor::Shell::Color::BLUE
-            result["conflicts"].each do |a|
-              say "\t\tC:\t#{a}", Thor::Shell::Color::RED
-            end
+        if result["conflicts"].size > 0
+          say "Conflicted changes:\n", Thor::Shell::Color::BLUE
+          result["conflicts"].each do |a|
+            say "\t\tC:\t#{a}", Thor::Shell::Color::RED
           end
-          log_end(0)
-        rescue SignalException
-          log_end(-1)
-          say "/nAborting"
-          exit(1)
         end
+        log_end(0)
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
       end
+    end
 
 
-      desc 'upload', 'Upload updated files'
-      method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    desc 'upload', 'Upload updated files'
+    method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
 
-      def upload(link=false, sync=false)
+    def upload(link=false, sync=false)
 
-        begin
-          verify_logged_in()
-          log_start(__method__,args,options)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
 
-          @project = Project.new(get_project_home)
+        @project = Project.new(get_project_home)
 
-          @files = Cnvrg::Files.new(@project.owner, @project.slug)
-          ignore = options[:ignore] || []
-          if !@project.update_ignore_list(ignore)
-            say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
-          end
-          result = @project.compare_idx
-          commit = result["result"]["commit"]
-          if !link
-            if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
-              log_end(0)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        ignore = options[:ignore] || []
+        if !@project.update_ignore_list(ignore)
+          say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
+        end
+        new_branch = options["new_branch"] || false
 
-              say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
-              exit(1)
-            end
-            say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
-          end
-          result = result["result"]["tree"]
-          if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+        result = @project.compare_idx(new_branch)
+        commit = result["result"]["commit"]
+        if !link
+          if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
             log_end(0)
-            say "Project is up to date", Thor::Shell::Color::GREEN
-            return true
+
+            say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
+            exit(1)
           end
-          update_count = 0
-          update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
-          successful_updates = []
-          successful_deletions = []
+
+          say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
+        end
+        result = result["result"]["tree"]
+        if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
+          log_end(0)
+          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
+          return true
+        end
+        update_count = 0
+        update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
+        successful_updates = []
+        successful_deletions = []
+        if options["verbose"]
           if update_total == 1
             say "Updating #{update_total} file", Thor::Shell::Color::BLUE
           else
             say "Updating #{update_total} files", Thor::Shell::Color::BLUE
           end
+        else
+          say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
 
-          # Start commit
+        end
 
-          commit_sha1 = @files.start_commit["result"]["commit_sha1"]
+        # Start commit
 
-          # upload / update
-          begin
-            (result["added"] + result["updated_on_local"]).each do |f|
-              puts f
-              absolute_path = "#{@project.local_path}/#{f}"
-              relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-              if File.directory?(absolute_path)
-                resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
-                if resDir
-                  update_count += 1
-                  successful_updates<< relative_path
-                end
+        commit_sha1 = @files.start_commit(new_branch)["result"]["commit_sha1"]
+
+        # upload / update
+        begin
+          (result["added"] + result["updated_on_local"]).each do |f|
+            absolute_path = "#{@project.local_path}/#{f}"
+            relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
+            if File.directory?(absolute_path)
+              resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
+              if resDir
+                update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              res = @files.upload_file(absolute_path, relative_path, commit_sha1)
+              if res
+                update_count += 1
+                successful_updates<< relative_path
               else
-                res = @files.upload_file(absolute_path, relative_path, commit_sha1)
-                if res
-                  update_count += 1
-                  successful_updates<< relative_path
-                else
-                  @files.rollback_commit(commit_sha1)
-                  log_end(1,"can't upload, Rolling Back all changes")
-                  say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
-                  exit(0)
-                end
+                @files.rollback_commit(commit_sha1)
+                log_end(1, "can't upload, Rolling Back all changes")
+                say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
+                exit(0)
               end
             end
+          end
 
-            # delete
-            result["deleted"].each do |f|
-              relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
-              if relative_path.end_with?("/")
-                if @files.delete_dir(f, relative_path, commit_sha1)
-                  update_count += 1
-                  successful_updates<< relative_path
-                end
-              else
-                if @files.delete_file(f, relative_path, commit_sha1)
-                  update_count += 1
-                  successful_updates<< relative_path
-                end
+          # delete
+          result["deleted"].each do |f|
+            relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
+            if relative_path.end_with?("/")
+              if @files.delete_dir(f, relative_path, commit_sha1)
+                update_count += 1
+                successful_updates<< relative_path
+              end
+            else
+              if @files.delete_file(f, relative_path, commit_sha1)
+                update_count += 1
+                successful_updates<< relative_path
               end
             end
-            log_end(0)
-
-          rescue SignalException
-            log_end(-1)
-            @files.rollback_commit(commit_sha1)
-            say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
-            exit(0)
-          rescue => e
-            log_end(1,e.message)
-            @files.rollback_commit(commit_sha1)
-            say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
-            exit(0)
           end
-          if update_count == update_total
-            res = @files.end_commit(commit_sha1)
-            if (Cnvrg::CLI.is_response_success(res, false))
-              # save idx
-              begin
-                @project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
+          log_end(0)
 
-                @project.update_idx_with_commit!(commit_sha1)
-              rescue => e
-                log_end(1,e.message)
-                @files.rollback_commit(commit_sha1)
-                say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
-                exit(1)
+        rescue SignalException
+          log_end(-1)
+          @files.rollback_commit(commit_sha1)
+          say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
+          exit(0)
+        rescue => e
+          log_end(1, e.message)
+          @files.rollback_commit(commit_sha1)
+          say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
+          exit(0)
+        end
+        if update_count == update_total
+          res = @files.end_commit(commit_sha1)
+          if (Cnvrg::CLI.is_response_success(res, false))
+            # save idx
+            begin
+              @project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
+
+              @project.update_idx_with_commit!(commit_sha1)
+            rescue => e
+              log_end(1, e.message)
+              @files.rollback_commit(commit_sha1)
+              say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
+              exit(1)
 
-              end
+            end
+            image = is_project_with_docker(Dir.pwd)
+            if image and image.is_docker
+              image.update_image_activity(commit_sha1, nil)
+            end
+            check = Helpers.checkmark()
 
-              say "Done", Thor::Shell::Color::BLUE
+            if options["verbose"]
+              say "#{check} Done", Thor::Shell::Color::BLUE
               if successful_updates.size >0
                 say "Updated:", Thor::Shell::Color::GREEN
                 suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
@@ -579,832 +859,2290 @@ module Cnvrg
                 say del.join("\n"), Thor::Shell::Color::GREEN
               end
               say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
-              log_end(0)
             else
-              @files.rollback_commit(commit_sha1)
-              log_end(1, "error. Rolling Back all changes")
-              say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
+              say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN unless options["sync"]
+
             end
+
+            log_end(0)
           else
+            @files.rollback_commit(commit_sha1)
             log_end(1, "error. Rolling Back all changes")
-
-            @files.rollback_commit(commit_sha1)
+            say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
           end
-        rescue SignalException
-          log_end(-1)
+        else
+          log_end(1, "error. Rolling Back all changes")
 
-          say "\nAborting",Thor::Shell::Color::BLUE
-          say "\nRolling back all changes",Thor::Shell::Color::BLUE
           @files.rollback_commit(commit_sha1)
-          exit(1)
         end
+      rescue
+        log_end(-1)
 
+        say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
+      rescue SignalException
+        log_end(-1)
+
+        say "\nAborting", Thor::Shell::Color::BLUE
+        say "\nRolling back all changes", Thor::Shell::Color::BLUE
+        @files.rollback_commit(commit_sha1)
+        exit(1)
       end
 
-      desc 'download', 'Download updated files'
+    end
 
-      def download
-        begin
-          verify_logged_in()
-          log_start(__method__,args,options)
-          project_home = get_project_home
-          @project = Project.new(project_home)
-          @files = Cnvrg::Files.new(@project.owner, @project.slug)
+    desc 'download', 'Download updated files'
+    method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+    method_option :sync, :type => :boolean, :aliases => ["--v"], :default => false
 
-          res = @project.compare_idx["result"]
-          result = res["tree"]
-          commit = res["commit"]
-          if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
-            say "Project is up to date", Thor::Shell::Color::GREEN
-            log_end(0)
-            return true
-          end
-          update_count = 0
-          update_total = result["updated_on_server"].size + result["conflicts"].size
+    def download
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        project_home = get_project_home
+        @project = Project.new(project_home)
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        new_branch = options["new_branch"] || false
+
+        res = @project.compare_idx(new_branch)["result"]
+        result = res["tree"]
+        commit = res["commit"]
+        if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
+          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
+          log_end(0)
+          return true
+        end
+        update_count = 0
+        update_total = result["updated_on_server"].size + result["conflicts"].size
 
-          successful_changes = []
-          if update_total ==1
-            say "Downloading #{update_total} file", Thor::Shell::Color::BLUE
-          else
-            say "Downloading #{update_total} files", Thor::Shell::Color::BLUE
 
-          end
+        successful_changes = []
+        if update_total ==1
+          say "Downloading #{update_total} file", Thor::Shell::Color::BLUE unless options["sync"]
+        elsif update_total == 0
+          say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
+          log_end(0)
+          return true
+        elsif options["verbose"]
+          say "Downloading #{update_total} files", Thor::Shell::Color::BLUE
+        else
+          say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
 
-          result["conflicts"].each do |f|
-            relative_path = f.gsub(/^#{@project.local_path}/, "")
-            if @files.download_file(f, relative_path, project_home, conflict=true)
-              successful_changes << relative_path
-            end
 
+        end
+
+        result["conflicts"].each do |f|
+          relative_path = f.gsub(/^#{@project.local_path}/, "")
+          if @files.download_file_s3(f, relative_path, project_home, conflict=true)
+            successful_changes << relative_path
           end
-          result["updated_on_server"].each do |f|
-            relative_path = f.gsub(/^#{@project.local_path}/, "")
-            if f.end_with? "/"
-              # dir
-              if @files.download_dir(f, relative_path, project_home)
-                successful_changes << relative_path
 
-              end
-            else
-              # blob
-              if @files.download_file(f, relative_path, project_home)
-                successful_changes << relative_path
-              end
-            end
+        end
+        result["updated_on_server"].each do |f|
+          relative_path = f.gsub(/^#{@project.local_path}/, "")
+          if f.end_with? "/"
+            # dir
+            if @files.download_dir(f, relative_path, project_home)
+              successful_changes << relative_path
 
+            end
+          else
+            # blob
+            if @files.download_file_s3(f, relative_path, project_home)
+              successful_changes << relative_path
+            end
           end
-          if update_total == successful_changes.size
-            # update idx with latest commit
-            @project.update_idx_with_commit!(commit)
 
-            say "Done. Downloaded:", Thor::Shell::Color::GREEN
+        end
+        if update_total == successful_changes.size
+          # update idx with latest commit
+          @project.update_idx_with_commit!(commit)
+          check = Helpers.checkmark()
+
+          if options["verbose"]
+            say "#{check} Done, Downloaded:", Thor::Shell::Color::GREEN
             say successful_changes.join("\n"), Thor::Shell::Color::GREEN
             say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
-            log_end(0)
+          else
+            say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
           end
-        rescue SignalException
-          log_end(-1)
-          say "\nAborting", Thor::Shell::Color::BLUE
-          if successful_changes.nil?
-            exit(1)
+
+
+          log_end(0)
+        end
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting", Thor::Shell::Color::BLUE
+        if successful_changes.nil?
+          exit(1)
+        end
+        successful_changes.each do |f|
+
+          abs_path = "#{@project.local_path}/#{f}"
+          filename = File.basename abs_path
+          say "revoking #{filename}"
+          if result["conflicts"].include? f
+            @files.revoke_download_file(abs_path, f, filename, true)
+          elsif result["updated_on_server"].include? f
+            if File.directory? abs_path
+              @files.revoke_download_dir(abs_path, f, project_home)
+            else
+              @files.revoke_download_file(project_home, abs_path, filename)
+            end
           end
-          successful_changes.each do |f|
+        end
+        exit(1)
+      end
+    end
 
-            abs_path = "#{@project.local_path}/#{f}"
-            filename = File.basename abs_path
-            say "revoking #{filename}"
-            if result["conflicts"].include? f
-              @files.revoke_download_file(abs_path,f,filename,true)
-            elsif result["updated_on_server"].include? f
-              if File.directory? abs_path
-                @files.revoke_download_dir(abs_path,f,project_home)
-              else
-                @files.revoke_download_file(project_home,abs_path,filename)
+
+    desc 'jump', 'jump to specific commit'
+
+    def jump(commit_sha1)
+      begin
+        verify_logged_in()
+        log_start(__method__, args, options)
+        project_home = get_project_home
+        @project = Project.new(project_home)
+        # say "Syncing existing project tree before jumping", Thor::Shell::Color::BLUE
+        current_commit = @project.last_local_commit
+        # if current_commit.eql? commit_sha1
+        #   say "Project is Updated", Thor::Shell::Color::GREEN
+        #   return
+        # end
+        @files = Cnvrg::Files.new(@project.owner, @project.slug)
+        #
+        # say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+        #
+        # download()
+        # upload()
+        # say "Done Syncing", Thor::Shell::Color::BLUE
+
+        response = @project.clone(false, commit_sha1)
+        successful_changes = []
+        if !response["result"]["tree"].nil?
+          idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
+          File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
+          current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
+
+          response["result"]["tree"].each do |f|
+            relative_path = f[0].gsub(/^#{@project.local_path}/, "")
+            if f[0].end_with? "/"
+              # dir
+              if @files.download_dir(f[0], relative_path, project_home)
+                current_tree.delete(relative_path[0, relative_path.size-1])
+                successful_changes << relative_path
+              end
+            else
+              # blob
+              if @files.download_file_s3(f[0], relative_path, project_home)
+                current_tree.delete(relative_path)
+
+                successful_changes << relative_path
               end
             end
           end
-          exit(1)
         end
+        FileUtils.rm_rf(current_tree)
+        say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
+        log_end(0)
+      rescue SignalException
+        log_end(-1)
+        exi(1)
       end
+    end
 
 
-      desc 'sync', 'Sync with remote server'
+    desc 'sync', 'Sync with remote server'
+    method_option :new_branch, :type => :boolean, :aliases => ["--nb"], :desc => "create new branch of commits"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
 
-      def sync
-        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
-        invoke :download
-        invoke :upload
-      end
-
-      # desc 'random', 'random'
-      # def random
-      #   say "Fun trivia game for taking a recess :-)", Thor::Shell::Color::BLUE
-      #   subject = ask("Pick a subject\n1.NBA\n2.American History\n3.Data Science\n")
-      #   file = "ds.txt"
-      #   case subject
-      #     when "1"
-      #       file = "nba.txt"
-      #     when "2"
-      #       file = "ah.txt"
-      #     when "3"
-      #       file = "ds.txt"
-      #   end
-      #   line = File.readlines(file).sample
-      #   q = line[0,line.index('?')+1]
-      #   a = line[line.index('?')+1,line.size]
-      #   answer = ask(q+"/n")
-      #
-      #
-      #
-      # end
-      # Run
-      #
-      desc 'exec CMD', 'Execute a process'
-      method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
-      method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
-      method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
-      method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
-      method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
-      method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
-
-      def exec(*cmd)
-        # LogJob.perform_async(cmd,options)
-        #
-          log = []
-          cpu_average =0
-          memory_average = 0
-          verify_logged_in()
-          log_start(__method__,args,options)
-          project_home = get_project_home
-          @project = Project.new(project_home)
-          sync_before = options["sync_before"]
-          sync_after = options["sync_after"]
-          print_log = options["log"]
-          title = options["title"]
-          email_notification = options["email_notification"]
-          upload_output = options["upload_output"]
-          time_to_upload = calc_output_time(upload_output)
+    def sync
+      if options["verbose"]
+      say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+      else
+        say 'Syncing project', Thor::Shell::Color::BLUE
+      end
 
-        begin
-          if sync_before
-            # Sync before run
-            say "Syncing project before running", Thor::Shell::Color::BLUE
-            say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
 
-            download()
-            upload()
-            say "Done Syncing", Thor::Shell::Color::BLUE
-          end
+      invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
+      invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
+      check = Helpers.checkmark()
 
-          start_commit = @project.last_local_commit
-          cmd = cmd.join("\s")
+      say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
 
-          say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
+    end
 
-          @exp = Experiment.new(@project.owner, @project.slug)
 
-          platform = RUBY_PLATFORM
-          machine_name = Socket.gethostname
-          begin
+    desc 'run cmd', 'Runs an experiment'
+    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
+    method_option :sync_before, :type => :boolean, :aliases => ["--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s"], :default => ""
+
+
+    def run(*cmd)
+      sync_before = options["sync_before"]
+      sync_after = options["sync_after"]
+      log = options["log"]
+      title = options["title"]
+      commit = options["commit"] || nil
+      email_notification = options["email_notification"]
+      upload_output = options["upload_output"]
+      local = options["local"]
+      schedule = options["schedule"]
+      if local
+        invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
+               :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit
+        return
+      else
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
+        instance_type = get_instance_type(instances)
+        invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
+               :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit
+        return
+      end
+    end
 
-            @exp.start(cmd, platform, machine_name, start_commit, title, email_notification)
-            unless @exp.slug.nil?
-              real = Time.now
-              exp_success = true
-              memory_total = []
-              cpu_total = []
-              start_loop = Time.now
-              begin
-                PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
-                  begin
-                    stdout.each do |line|
-                      cur_time = Time.now
-                      monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
-                      monitor_by = monitor.split(" ")
-                      memory = monitor_by[2]
-                      cpu = monitor_by[1]
-                      memory_total << memory.to_f
-                      cpu_total << cpu.to_f
-                      real_time= Time.now-real
-
-                      cur_log = {time: cur_time,
-                                 message: line,
-                                 type: "stdout",
-                                 real: real_time}
-                      if print_log
-                        puts cur_log
-                      end
-                      log << cur_log
 
-                      begin
-                        if time_to_upload !=0
-                          if time_to_upload <= Time.now - start_loop
-                            #upload current log
-                            cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
-                            memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+    desc '', ''
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
+
+    def exec(*cmd)
+      log = []
+      cpu_average =0
+      memory_average = 0
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      working_dir = Dir.pwd
+      image = is_project_with_docker(working_dir)
+
+
+      sync_before = options["sync_before"]
+      sync_after = options["sync_after"]
+      print_log = options["log"]
+      title = options["title"]
+      commit = options["commit"] || nil
+      email_notification = options["email_notification"]
+      upload_output = options["upload_output"]
+      time_to_upload = calc_output_time(upload_output)
+      project_home = get_project_home
+      @project = Project.new(project_home)
+
+      is_new_branch = @project.compare_commit(commit)
+      begin
+        if !commit.nil? and !commit.empty?
+          invoke :jump, [commit], []
+        else
+          if sync_before
+            # Sync before run
+            invoke :sync, [], :new_branch => is_new_branch
+          end
+        end
 
-                            @exp.upload_temp_log(log, cpu_average, memory_average)
-                            log = []
-                            start_loop = Time.now
-                          end
 
-                        end
-                      rescue
-                        say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
-                      end
+        if image and image.is_docker
+          container= image.get_container
+          if !container
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+          exec_args = args.flatten.join(" ")
+          exec_options = options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+          command_to_run = cmd.join(" ")
+          command = ["/bin/bash", "-lc", "cnvrg exec_docker #{exec_options} #{command_to_run} #{exec_args}"]
+          puts container.exec(command, tty: true)
+          container.stop()
+          exit(0)
+        end
 
-                    end
+        start_commit = @project.last_local_commit
+        cmd = cmd.join("\s")
 
+        say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
 
-                    if stderr
+        @exp = Experiment.new(@project.owner, @project.slug)
 
-                      stderr.each do |err|
+        platform = RUBY_PLATFORM
+        machine_name = Socket.gethostname
+        begin
+          machine_activity = @exp.get_machine_activity(Dir.pwd)
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          unless @exp.slug.nil?
+            real = Time.now
+            exp_success = true
+            memory_total = []
+            cpu_total = []
+            start_loop = Time.now
+            begin
+              PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
+                begin
+                  stdout.each do |line|
+                    cur_time = Time.now
+                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
+                    monitor_by = monitor.split(" ")
+                    memory = monitor_by[2]
+                    cpu = monitor_by[1]
+                    memory_total << memory.to_f
+                    cpu_total << cpu.to_f
+                    real_time= Time.now-real
+
+                    cur_log = {time: cur_time,
+                               message: line,
+                               type: "stdout",
+                               real: real_time}
+                    if print_log
+                      puts cur_log
+                    end
+                    log << cur_log
+
+                    begin
+                      if time_to_upload !=0
+                        if time_to_upload <= Time.now - start_loop
+                          #upload current log
+                          cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                          memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+
+                          @exp.upload_temp_log(log, cpu_average, memory_average)
+                          log = []
+                          start_loop = Time.now
+                        end
 
-                        log << {time: Time.now, message: err, type: "stderr"}
                       end
+                    rescue
+                      say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
                     end
 
-                  rescue Errno::EIO => e
-                    break
-                  rescue Errno::ENOENT
-                    log_end(1, "command #{cmd} isn't valid")
+                  end
+
 
-                    exp_success = false
+                  if stderr
 
-                    say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
-                  rescue PTY::ChildExited
-                    log_end(1, "proccess exited")
-                    exp_success = false
-                    say "The process exited!", Thor::Shell::Color::RED
-                  rescue => e
-                    log_end(1,e.message)
-                  end
-                  ::Process.wait pid
-                  cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
-                  memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
-                  exit_status = $?.exitstatus
-                  if $?.exitstatus != 0
-                    exp_success = false
+                    stderr.each do |err|
+
+                      log << {time: Time.now, message: err, type: "stderr"}
+                    end
                   end
-                  if !exp_success
-                    if !Cnvrg::Helpers.internet_connection?
-                      wait_offline = agree "Seems like you're offline, wait until your'e back online?",limited_to: ['y', 'n'], default: 'n'
-                      if wait_offline
-                        say "Waiting until your'e online..", Thor::Shell::Color::BLUE
-                        while !Cnvrg::Helpers.internet_connection?
-                        end
-                        exit_status = 0
-                      else
-                        say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
-                        log_end(1,"experiment has failed,computer is offline")
-                        exit(0)
+
+                rescue Errno::EIO => e
+                  puts e
+                    # break
+                rescue Errno::ENOENT
+                  log_end(1, "command #{cmd} isn't valid")
+
+                  exp_success = false
+
+                  say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+                rescue PTY::ChildExited
+                  log_end(1, "proccess exited")
+                  exp_success = false
+                  say "The process exited!", Thor::Shell::Color::RED
+                rescue => e
+                  puts e
+                  log_end(1, e.message)
+                end
+                ::Process.wait pid
+                cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+                exit_status = $?.exitstatus
+                if $?.exitstatus != 0
+                  exp_success = false
+                end
+                if !exp_success
+                  if !Cnvrg::Helpers.internet_connection?
+                    wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+                    if wait_offline
+                      say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+                      while !Cnvrg::Helpers.internet_connection?
                       end
+                      exit_status = 0
                     else
+                      say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
+                      log_end(1, "experiment has failed,computer is offline")
+                      exit(0)
+                    end
+                  else
 
                     end_commit = @project.last_local_commit
                     res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
                     say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
-                    log_end(1,"experiment has failed")
+                    log_end(1, "experiment has failed")
                     exit(0)
-                    end
-
-                  end
-                  if sync_after
-                    say "Syncing project after running", Thor::Shell::Color::BLUE
-                    # Sync after run
-                    download()
-                    upload()
-                    say "Done Syncing", Thor::Shell::Color::BLUE
                   end
-                  end_commit = @project.last_local_commit
 
-                  res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-                  check = Helpers.checkmark()
-                  say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
-                  log_end(0)
                 end
-              rescue =>e
-                log_end(1,e.message)
-                say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
-                exit(1)
-              end
+                if sync_after
+                  # Sync after run
+                  if !commit.nil?
+                    invoke :sync, [], :new_branch => true
 
+                  else
+                    invoke :sync, [], :new_branch => true
+                  end
+                end
+                end_commit = @project.last_local_commit
 
+                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                check = Helpers.checkmark()
+                say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
+                log_end(0)
+              end
+            rescue => e
+              puts e
+              log_end(1, e.message)
+              if container
+                container.stop()
+              end
+              say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
+              exit(1)
             end
 
+
           end
-        rescue SignalException
-          exit_status =  -1
-          log_end(-1)
-          end_commit = @project.last_local_commit
 
-          res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
-          say "\nAborting"
+        end
+      rescue SignalException
+        exit_status = -1
+        log_end(-1)
+        end_commit = @project.last_local_commit
 
-          exit(1)
+        res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+        if container
+          container.stop()
         end
+        say "\nAborting"
+
+        exit(1)
       end
+    end
+
+    desc '', ''
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
+
+    def exec_docker(*cmd)
+      log = []
+      cpu_average =0
+      memory_average = 0
+      verify_logged_in()
+      log_start(__method__, args, options)
+      project_home = "/home/ds/notebooks"
+      @project = Project.new(project_home)
+      sync_before = options["sync_before"]
+      sync_after = options["sync_after"]
+      print_log = options["log"]
+      title = options["title"]
+      email_notification = options["email_notification"]
+      upload_output = options["upload_output"]
+      time_to_upload = calc_output_time(upload_output)
+      @image = is_project_with_docker(project_home)
 
-      desc 'run_notebook', 'Starts a new notebook environment'
-      method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
-      method_option :docker, :type => :boolean, :aliases => ["-d", "--d"], :default => true
-      method_option :image_name, :type => :string, :aliases => ["-i", "--i"], :default => ""
 
-      def run_notebook
+      begin
+        start_commit = @project.last_local_commit
+        cmd = cmd.join("\s")
+
+        say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
 
+        @exp = Experiment.new(@project.owner, @project.slug)
+        machine_activity = @exp.get_machine_activity(project_home)
+
+        platform = RUBY_PLATFORM
+        machine_name = Socket.gethostname
         begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
-          cur_path = Dir.pwd
-          notebook_dir = options["notebook_dir"]
-          if notebook_dir.empty?
-            notebook_dir = cur_path
-          else
-            notebook_dir = cur_path+ notebook_dir
-          end
-          say "Linking notebook directory to: #{notebook_dir}", Thor::Shell::Color::BLUE
-          docker = options["docker"]
-          try_again = true
 
+          @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
+          unless @exp.slug.nil?
+            real = Time.now
+            exp_success = true
+            memory_total = []
+            cpu_total = []
+            start_loop = Time.now
+            begin
+              PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
+                begin
+                  stdout.each do |line|
+                    cur_time = Time.now
+                    monitor = %x{ps aux|awk  '{print $2,$3,$4}'|grep #{pid} }
+                    monitor_by = monitor.split(" ")
+                    memory = monitor_by[2]
+                    cpu = monitor_by[1]
+                    memory_total << memory.to_f
+                    cpu_total << cpu.to_f
+                    real_time= Time.now-real
+
+                    cur_log = {time: cur_time,
+                               message: line,
+                               type: "stdout",
+                               real: real_time}
+                    if print_log
+                      puts cur_log
+                    end
+                    log << cur_log
+
+                    begin
+                      if time_to_upload !=0
+                        if time_to_upload <= Time.now - start_loop
+                          #upload current log
+                          cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                          memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+
+                          @exp.upload_temp_log(log, cpu_average, memory_average)
+                          log = []
+                          start_loop = Time.now
+                        end
 
-          if docker
-            docker_path = verify_software_installed("docker")
-            image_name = options["image_name"]
-            if image_name.empty?
-              images_list = `#{docker_path} images`
-              if images_list.empty? or images_list.nil?
-                say "you don't have any images to run", Thor::Shell::Color::BLUE
-              else
-                say "Choose image name to run as a container\n", Thor::Shell::Color::BLUE
-                say images_list
-                image_name = ask("\nwhat is the image name?\n")
-              end
+                      end
+                    rescue
+                      say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
+                    end
+
+                  end
 
-            end
-            while (try_again) do
-              run_docker = `#{docker_path} run -d -p 8888:8888  -v #{notebook_dir}:/home/ds/notebooks -t -i #{image_name} 2>&1`
-              if !run_docker.match(/[a-z0-9]{64}/).nil? and !run_docker.include? "Error"
-                container_id = get_container_id()
-                sleep(3)
-                logs = `docker logs #{container_id}`
-                url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
-                if !url.empty?
-                  check = Helpers.checkmark()
-
-                  say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
-                  try_again= false
-                else
-                  say "Couldn't start notebook server", Thor::Shell::Color::RED
-                  log_end(1, "can't start notebook server")
-                  exit(1)
-                end
 
-              elsif run_docker.include? "port is already allocated"
-                say "Couldn't start notebook with port 8888, port is taken", Thor::Shell::Color::RED
-                port_container = `#{docker_path} ps |grep 8888 |awk '{print $1}'`.strip!
-                stop = agree "There is another running notebook server: #{port_container}, do you want to stop it?", limited_to: ['y', 'n'], default: 'y'
-                if stop == "y"
-                  did_stop = system("#{docker_path} stop #{port_container}")
-                  if !did_stop
-                    say "Couldn't stop notebook server: #{port_container}", Thor::Shell::Color::RED
-                    log_end(1,"can't stop notebook server")
-                    exit(1)
+                  if stderr
 
+                    stderr.each do |err|
+
+                      log << {time: Time.now, message: err, type: "stderr"}
+                    end
                   end
-                else
-                  logs = `#{docker_path} logs #{port_container}`
-                  url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
-                  say "Done, your notebook server is: #{url}", Thor::Shell::Color::BLUE
-                  log_end(0)
-                  exit(1)
+
+                rescue Errno::EIO => e
+                  break
+                rescue Errno::ENOENT
+                  log_end(1, "command #{cmd} isn't valid")
+
+                  exp_success = false
+
+                  say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
+                rescue PTY::ChildExited
+                  log_end(1, "proccess exited")
+                  exp_success = false
+                  say "The process exited!", Thor::Shell::Color::RED
+                rescue => e
+                  log_end(1, e.message)
                 end
-              else
-                log_end(1, "can;t start notebook server")
-                say "Couldn't start notebook server", Thor::Shell::Color::RED
-                exit(1)
+                ::Process.wait pid
+                cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
+                memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
+                exit_status = $?.exitstatus
+                if $?.exitstatus != 0
+                  exp_success = false
+                end
+                if !exp_success
+                  if !Cnvrg::Helpers.internet_connection?
+                    wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+                    if wait_offline
+                      say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+                      while !Cnvrg::Helpers.internet_connection?
+                      end
+                      exit_status = 0
+                    else
+                      say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
+                      log_end(1, "experiment has failed,computer is offline")
+                      exit(0)
+                    end
+                  else
+
+                    end_commit = @project.last_local_commit
+                    res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                    @image.update_image_activity(@project.last_local_commit, @exp.slug)
+                    say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
+                    log_end(1, "experiment has failed")
+                    exit(0)
+                  end
+
+                end
+                if sync_after
+                  say "Syncing project after running", Thor::Shell::Color::BLUE
+                  # Sync after run
+                  download()
+                  upload()
+                  say "Done Syncing", Thor::Shell::Color::BLUE
+                end
+                end_commit = @project.last_local_commit
 
+                res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+                @image.update_image_activity(@project.last_local_commit, @exp.slug)
+
+                check = Helpers.checkmark()
+                say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
+                log_end(0)
               end
-            end
-          else
-            jup =verify_software_installed("jupyter-notebook")
-            logs = `#{jup} --no-browser --ip=0.0.0.0  --notebook-dir=#{notebook_dir}`
-            url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
-            if !url.empty?
-              check = Helpers.checkmark()
-
-              say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
-              log_end(0)
-            else
-              say "Couldn't start notebook server", Thor::Shell::Color::RED
-              log_end(1,"can't start notebook server")
+            rescue => e
+              log_end(1, e.message)
+              say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
               exit(1)
             end
 
+
           end
-        rescue SignalException
-          log_end(-1)
-          say "Aborting"
-          exit(1)
+
+        end
+      rescue SignalException
+        exit_status = -1
+        log_end(-1)
+        end_commit = @project.last_local_commit
+        if !@exp.nil?
+
+          res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
         end
 
+        say "\nAborting"
 
+        exit(1)
       end
+    end
 
-      desc 'install_notebook_libraries', 'Starts a new notebook environment'
-      method_option :commit_name, :type => :string, :aliases => ["-c", "--c"], :default => "", :desc => "commit changes to notebook server when finished"
+    desc '', ''
+    method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
+    method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
+    method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
+    method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
+    method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
+    method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => nil
+
+    def exec_remote(*cmd)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir
+      begin
 
-      def install_notebook_libraries
-        begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
-          docker_path = verify_software_installed("docker")
-          container_id = get_container_id
-          say "Opening shell in notebook server\nYou can run pip install [library] to install more tools\ntype exit to finish", Thor::Shell::Color::BLUE
-          system("#{docker_path} exec -it #{container_id} bash")
-          commit_name = options["commit_name"]
-          if !commit_name.empty?
-            return commit_notebook(commit_name)
+        instance_type = options["machine_type"] || nil
+        schedule = options["schedule"] || ""
+        if schedule.start_with? 'in'
+          time = schedule.split(" ")
+
+          local = Time.now.localtime
+          if time[2].downcase().start_with? "min"
+            new = local + (time[1].to_i * 60)
+          elsif time[2].downcase().start_with? "hours"
+            new = local + (time[1].to_i * 3600)
+          elsif time[2].downcase().start_with? "days"
+            new = local + (time[1].to_i * 3600 * 24)
+          else
+            say "Could not undersatnd when to schedule experiment", Thor::Shell::Color::RED
+            exit(1)
           end
-        rescue SignalException
-          log_End(-1)
-          say "/nAborting"
-          exit(1)
+          new_time = new.to_s
+          new_time = new_time[0, new_time.size-6] #remove timezone
+          schedule = "at #{new_time}"
+        end
+
+
+        options_hash = Hash[options]
+        options_hash.except!("schedule", "machine_type")
+        exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
+        command = "#{exec_options} --uo=1m #{cmd.flatten.join(" ")}"
+        commit_to_run = options["commit"] || nil
+        if !schedule.nil? and !schedule.empty?
+
+          local_timestamp = get_schedule_date
+
+        end
+        project = Project.new(working_dir)
+
+
+        image = is_project_with_docker(working_dir)
+        if !image or !image.is_docker
+          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
+          if default
+            image = Images.new(working_dir, "cnvrg")
+            image_slug = image.image_slug
+          else
+            exit(0)
+          end
+        else
+          image_slug = image.image_slug
         end
 
+
+
+        invoke :sync, [], []
+
+        say "Running remote experiment", Thor::Shell::Color::BLUE
+        exp = Experiment.new(project.owner, project.slug)
+
+        res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
+        if Cnvrg::CLI.is_response_success(res)
+
+          # if res["result"]["machine"] == -1
+          #   say "There are no available machines", Thor::Shell::Color::BLUE
+          #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
+          #   if create
+          #     res = Cnvrg::API.request("users/#{image.owner}/machines/list", 'GET')
+          #     if Cnvrg::CLI.is_response_success(res)
+          #       instance_type = machine_options(res["result"]["aws_options"])
+          #       if @image.new_machine(instance_type)
+          #         say "Running remote experiment", Thor::Shell::Color::BLUE
+          #
+          #         # res = image.exec_remote(exec_args, exec_options, project.last_local_commit)
+          #         # if Cnvrg::CLI.is_response_success(res)
+          #
+          #           check = Helpers.checkmark()
+          #           say "#{check} Finished successfuly", Thor::Shell::Color::GREEN
+          #           exit(0)
+          #         # end
+          #       end
+          #     else
+          #       say "No machines are avilable", Thor::Shell::Color::RED
+          #       exit(0)
+          #     end
+          #
+          #
+          #   else
+          #     say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
+          #     exit(1)
+          #
+          #   end
+          # else
+          check = Helpers.checkmark()
+          say "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}", Thor::Shell::Color::GREEN
+
+          exit(0)
+          # end
+        end
+
+
+      rescue SignalException
+        exit_status = -1
+        log_end(-1)
+        end_commit = project.last_local_commit
+
+        res = @exp.end(log, exit_status, end_commit, "", "")
+        say "\nAborting"
+
+        exit(1)
+      end
+    end
+
+    desc 'deploy', 'Deploys model to production'
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => ""
+    method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
+
+    def deploy(file_to_run, function)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir
+      begin
+
+        instance_type = options["machine_type"] || nil
+        schedule = options["schedule"] || ""
+
+
+
+        if !schedule.nil? and !schedule.empty?
+
+          local_timestamp = get_schedule_date
+
+        end
+        project = Project.new(working_dir)
+        commit_to_run = options["commit"] || nil
+
+
+        image = is_project_with_docker(working_dir)
+        if !image or !image.is_docker
+          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
+          if default
+            image = Images.new(working_dir, "cnvrg")
+            image_slug = image.image_slug
+          else
+            exit(0)
+          end
+        else
+          image_slug = image.image_slug
+        end
+
+
+
+        invoke :sync, [], []
+
+        res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
+
+        if Cnvrg::CLI.is_response_success(res)
+
+          # if res["result"]["machine"] == -1
+          #   say "There are no available machines", Thor::Shell::Color::BLUE
+          #   create = yes? "create new machine?", Thor::Shell::Color::YELLOW
+          #   if create
+          #     res = Cnvrg::API.request("users/#{image.owner}/machines/list", 'GET')
+          #     if Cnvrg::CLI.is_response_success(res)
+          #       instance_type = machine_options(res["result"]["aws_options"])
+          #       if @image.new_machine(instance_type)
+          #         say "Running remote experiment", Thor::Shell::Color::BLUE
+          #
+          #         # res = image.exec_remote(exec_args, exec_options, project.last_local_commit)
+          #         # if Cnvrg::CLI.is_response_success(res)
+          #
+          #           check = Helpers.checkmark()
+          #           say "#{check} Finished successfuly", Thor::Shell::Color::GREEN
+          #           exit(0)
+          #         # end
+          #       end
+          #     else
+          #       say "No machines are avilable", Thor::Shell::Color::RED
+          #       exit(0)
+          #     end
+          #
+          #
+          #   else
+          #     say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
+          #     exit(1)
+          #
+          #   end
+          # else
+          check = Helpers.checkmark()
+          say "#{check} Deployment process is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/deploys/show/#{res["result"]["deploy_slug"]}", Thor::Shell::Color::GREEN
+
+          exit(0)
+          # end
+        end
+
+
+      rescue SignalException
+        exit_status = -1
+        log_end(-1)
+        end_commit = project.last_local_commit
+
+        res = @exp.end(log, exit_status, end_commit, "", "")
+        say "\nAborting"
+
+        exit(1)
+      end
+    end
+
+    method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
+    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :local, :type => :boolean, :aliases => ["--l"], :default => false
+    method_option :small, :type => :boolean, :aliases => ["--sm"], :default => false
+    method_option :medium, :type => :boolean, :aliases => ["--med"], :default => false
+    method_option :large, :type => :boolean, :aliases => ["--lg"], :default => false
+    method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
+    method_option :gpuxl, :type => :boolean, :aliases => ["--gxl"], :default => false
+    desc 'starts a notebook session', 'starts a notebook session remotely or locally'
+
+    def notebook()
+      local = options["local"]
+      notebook_dir = options["notebook_dir"]
+      kernel = options["kernel"]
+
+      if local
+        invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel
+        return
+      else
+        instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"], "gpu" => options["gpu"], "gpuxl" => options["gpuxl"]}
+        instance_type = get_instance_type(instances)
+
+        invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type
+        return
+
+      end
+
+
+    end
+
+    desc 'remote_notebook', 'run notebook server on remote server'
+    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :machine_type, :type => :string, :aliases => ["--i", "-i"], :default => ""
+    method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
+
+    def remote_notebook()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      verify_software_installed("docker")
+
+      working_dir = is_cnvrg_dir()
+      notebook_dir = options["notebook_dir"]
+      instance_type = options["machine_type"] || nil
+      kernel = options["kernel"] || nil
+
+
+      begin
+
+        @image = is_project_with_docker(working_dir)
+        if !@image.is_docker
+          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          exit(0)
+        end
+
+        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+
+        download()
+        upload()
+
+
+        res = @image.remote_notebook(notebook_dir, instance_type, kernel)
+        if Cnvrg::CLI.is_response_success(res)
+          if res["result"]["machine"] == -1
+            say "There are no available machines", Thor::Shell::Color::BLUE
+            create = yes? "create new machine?", Thor::Shell::Color::YELLOW
+            if create
+              res = Cnvrg::API.request("users/#{@image.owner}/machines/list", 'GET')
+              if Cnvrg::CLI.is_response_success(res)
+                instance_type = machine_options(res["result"]["aws_options"])
+                if @image.new_machine(instance_type)
+                  res = @image.remote_notebook(notebook_dir, instance_type, kernel)
+                  if Cnvrg::CLI.is_response_success(res)
+                    url = res["result"]["url"]
+                    if !url.nil? and !url.empty?
+                      check = Helpers.checkmark()
+
+                      say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
+                      Launchy.open(url)
+                    else
+                      say "Couldn't run notebook server", Thor::Shell::Color::RED
+                    end
+                    exit(0)
+                  end
+                end
+              else
+                say "No machines are avilable", Thor::Shell::Color::RED
+                exit(0)
+              end
+
+
+            else
+              say "Can't execute command on remote machine with local image", Thor::Shell::Color::RED
+              exit(1)
+
+            end
+          else
+            note_url = res["result"]["notebook_url"]
+            @image.set_note_url(note_url)
+            check = Helpers.checkmark()
+            say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
+            # Launchy.open(url)
+
+            exit(0)
+          end
+        end
+
+
+      rescue SignalException
+        exit_status = -1
+        log_end(-1)
+        end_commit = @project.last_local_commit
+
+        res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
+        say "\nAborting"
+
+        exit(1)
+      end
+    end
+
+    desc 'search_libraries', 'search if  libraries installed'
+
+    def search_libraries(library)
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+        project_dir = is_cnvrg_dir()
+
+        image = is_project_with_docker(project_dir)
+        if image and image.is_docker
+          container= image.get_container
+          if !container
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+
+        say "Searching for #{library}", Thor::Shell::Color::BLUE
+        pip_arr = image.get_installed_packages("python")
+        pip_arr = pip_arr.map(&:downcase)
+        check = Helpers.checkmark()
+        if !(p = pip_arr.map { |x| x.split("==")[0] }.index(library.downcase)).nil?
+
+          say "#{check} Found it!", Thor::Shell::Color::GREEN
+
+
+          printf "%-40s %-30s\n", "#{pip_arr[p].split("==")[0]}", "#{pip_arr[p].split("==")[1]}"
+
+        else
+          dpkg_arr = image.get_installed_packages("system")
+          dpkg_arr = dpkg_arr.map(&:downcase)
+          if !(p = dpkg_arr.map { |x| x.split("==")[0] }.index(library.downcase)).nil?
+            say "#{check} Found!", Thor::Shell::Color::GREEN
+
+            printf "%-40s %-30s\n", "#{dpkg_arr[p].split("==")[0]}", "#{dpkg_arr[p].split("==")[1]}"
+          else
+            say "Couldn't find #{library}, run cnvrg install_libraries to install", Thor::Shell::Color::RED
+            exit(1)
+          end
+
+
+        end
+      rescue => e
+        puts e
+        log_end(-1)
+        say "Error occurd, Aborting"
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_end(-1)
+        if container
+          container.stop()
+        end
+        say "Aborting"
+        exit(1)
+      end
+    end
+
+    desc 'show_libraries', 'show system libraries installed'
+    method_option :system, :type => :boolean, :aliases => ["-s", "--s"], :default => false, :desc => "show also system libraries installed"
+
+    def show_libraries
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+        system = options["system"] || false
+
+
+        project_dir = is_cnvrg_dir()
+
+        image = is_project_with_docker(project_dir)
+        if image and image.is_docker
+          container = image.get_container
+          if !container
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+
+        say "Showing python installed libraries", Thor::Shell::Color::BLUE
+        pip_arr = image.get_installed_packages("python")
+        printf "%-40s %-30s\n", "name", "version"
+        printf "%-40s %-30s\n", "====", "======="
+
+        pip_arr.each do |p|
+
+          printf "%-40s %-30s\n", "#{p.split("==")[0]}", "#{p.split("==")[1]}"
+        end
+        if system
+
+          say "Showing system installed libraries", Thor::Shell::Color::BLUE
+          dpkg_arr = image.get_installed_packages("system")
+          printf "%-40s %-30s\n", "name", "version"
+          printf "%-40s %-30s\n", "====", "======="
+          dpkg_arr.each do |p|
+            printf "%-40s %-30s\n", "#{p.split("==")[0]}", "#{p.split("==")[1]}"
+
+          end
+        end
+      rescue => e
+        puts e
+        log_end(-1)
+        say "Error occurd, Aborting"
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_end(-1)
+        say "Aborting"
+        exit(1)
+      end
+    end
+
+
+    desc 'run_notebook', 'Starts a new notebook environment'
+    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
+    method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+
+    def run_notebook
+
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+
+        project_dir = is_cnvrg_dir()
+
+        notebook_dir = options["notebook_dir"]
+        remote = options["remote"] || false
+        kernel = options["kernel"] || ""
+
+
+        if notebook_dir.empty?
+          notebook_dir = project_dir
+        else
+
+          notebook_dir = project_dir+ notebook_dir
+        end
+        say "Linking notebook directory to: #{File.basename notebook_dir}", Thor::Shell::Color::BLUE
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker and !remote
+          container= image.get_container
+          if !container
+
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+        if options["verbose"]
+          say "Syncing project before running", Thor::Shell::Color::BLUE
+          say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
+        end
+        @project = Project.new(project_dir)
+
+        start_commit = @project.last_local_commit
+
+        if (note_slug=image.note_slug)
+          say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
+          new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
+          if !new
+            exit(0)
+          end
+
+        end
+        invoke :sync, [], :verbose => options["verbose"]
+        say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
+        #replace url
+        base_url = get_base_url()
+
+        local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
+        command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
+        container.exec(command, tty: true)
+        container.stop()
+        container.start()
+
+        @note = Experiment.new(@project.owner, @project.slug)
+        port = image.container_port()
+
+        command = ["/bin/bash", "-lc", "jupyter notebook list"]
+        list = container.exec(command, tty: true)[0]
+        if list.empty? or list.nil?
+          say "Couldn't start notebook server", Thor::Shell::Color::RED
+          log_end(1, "can't start notebook server")
+          exit(1)
+        end
+
+        url =  list[list.size-1].split(" ::")
+        token = url[0][(url[0].index("=")+1)..-1]
+
+        # machine_activity = @note.get_machine_activity(project_dir)
+
+
+        slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
+        image.set_note_url(slug)
+        note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
+
+
+        if !note_url.empty?
+          check = Helpers.checkmark()
+
+          say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
+          Launchy.open(note_url)
+        else
+          say "Couldn't start notebook server", Thor::Shell::Color::RED
+          log_end(1, "can't start notebook server")
+          exit(1)
+        end
+
+          # jup =verify_software_installed("jupyter-notebook")
+          # logs = `#{jup} --no-browser --ip=0.0.0.0  --notebook-dir=#{notebook_dir}`
+          # url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
+          # if !url.empty?
+          #   check = Helpers.checkmark()
+          #
+          #   say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
+          #   log_end(0)
+          # else
+          #   say "Couldn't start notebook server", Thor::Shell::Color::RED
+          #   log_end(1, "can't start notebook server")
+          #   exit(1)
+          # end
+      rescue => e
+        log_end(-1)
+        puts e
+        say "Error occurd, aborting", Thor::Shell::Color::RED
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_end(-1)
+        if container
+          container.stop()
+        end
+        say "Aborting"
+        exit(1)
+      end
+
+
+    end
+
+    desc 'notebook_stop', 'Starts a new notebook environment'
+    method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
+    method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
+    method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
+
+    def notebook_stop
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+        remote = options["remote"] || false
+        project_dir = is_cnvrg_dir()
+
+
+        say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
+        invoke :sync, [], :verbose => options["verbose"]
+
+        say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
+
+
+        @project = Project.new(project_dir)
+
+        end_commit = @project.last_local_commit
+
+        @note = Experiment.new(@project.owner, @project.slug)
+
+        slug = get_note_url(project_dir)
+        @note.end_notebook_session(slug, end_commit)
+
+
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker and !remote
+          container = image.get_container(true)
+          if !container
+            check = Helpers.checkmark()
+
+            say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
+            exit(0)
+          end
+
+
+          say "Stopping notebook server...", Thor::Shell::Color::BLUE
+
+          check = Helpers.checkmark()
+          image.remove_note_slug()
+          container.stop()
+
+          say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
+          log_end(0)
+
+          exit(0)
+        elsif remote
+          say "Stopping notebook server...", Thor::Shell::Color::BLUE
+          check = Helpers.checkmark()
+
+          say "#{check} Notebook server stopped successfully", Thor::Shell::Color::GREEN
+          log_end(0)
+          exit(0)
+        end
+      rescue => e
+        puts e
+        log_end(-1)
+        say "Error occurd, aborting"
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_end(-1)
+        say "Aborting"
+        exit(1)
+      end
+
+
+    end
+
+
+    desc 'install_system_libraries', 'Install libraries'
+
+    def install_system_libraries(*command_to_run)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker
+          container = image.get_container
+          if !container
+
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+
+        command_to_run = command_to_run.join(" ")
+        say "Running #{command_to_run} in container", Thor::Shell::Color::BLUE
+        command = ["/bin/bash", "-lc", "#{command_to_run}"]
+        res = container.exec(command, tty: false)
+        say res[0].join("\n")
+        checks = Helpers.checkmark()
+        say "Updating image", Thor::Shell::Color::BLUE
+
+        image.create_custom_image("")
+        say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
+        container.stop()
+
+        log_end(0)
+      rescue => e
+        log_end(-1)
+        puts e
+        say "Error occurd, aborting"
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_End(-1)
+        if container
+          container.stop()
+        end
+        say "\nAborting"
+        exit(1)
+      end
+
+    end
+
+    desc 'install_libraries', 'Install libraries'
+    method_option :requirement, :type => :string, :aliases => ["-r", "--r"], :default => "", :desc => "Install from the given requirements file"
+
+    def install_python_libraries(*lib)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        verify_software_installed("docker")
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker
+          container = image.get_container
+          if !container
+
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+        req_file = options["requirement"] || nil
+        if lib.nil? and not req_file.nil?
+          if not File.exist? req_file
+            say "Couldn't find #{req_file}", Thor::Shell::Color::RED
+            exit(1)
+
+          end
+          command_to_run = "pip install -r #{req_file}"
+
+        else
+          command_to_run = lib.join(" ")
+
+        end
+        say "Running #{command_to_run} in container", Thor::Shell::Color::BLUE
+        command = ["/bin/bash", "-lc", "#{command_to_run}"]
+        res = container.exec(command, tty: false)
+        say res[0].join("\n")
+        checks = Helpers.checkmark()
+        say "Updating image", Thor::Shell::Color::BLUE
+
+        image.create_custom_image("")
+        say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
+        container.stop()
+
+        log_end(0)
+      rescue => e
+        log_end(-1)
+        puts e
+        say "Error occurd, aborting"
+        if container
+          container.stop()
+        end
+      rescue SignalException
+        log_End(-1)
+        if container
+          container.stop()
+        end
+        say "\nAborting"
+        exit(1)
+      end
+
+    end
+
+    desc 'commit_notebook', 'commit notebook changes to create a new notebook image'
+
+    def commit_image
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      verify_software_installed("docker")
+
+      begin
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker
+          container= image.get_container
+          if !container
+
+            say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
+            exit(1)
+          end
+        else
+          say "Project is not configured with any image", Thor::Shell::Color::RED
+          exit(1)
+
+        end
+        project_home = get_project_home
+        @project = Project.new(project_home)
+        last_local_commit = @project.last_local_commit
+        say "Commiting container into image", Thor::Shell::Color::BLUE
+        new_image_name = "#{@project.slug}#{last_local_commit}:latest"
+        image.update_image(new_image_name, container)
+        new_image = container.commit('repo' => "#{@project.slug}#{last_local_commit}", 'tag' => "lastest")
+        checks = Helpers.checkmark()
+        say "#{checks} Done, image was updated", Thor::Shell::Color::GREEN
+        log_end(0)
+        return new_image.id
+      rescue => e
+        puts e
+
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc 'sync_image', 'sync current container into image, and push it to cnvrg repository'
+    method_option :is_public, :type => :boolean, :aliases => ["-p", "--p"], :default => false, :desc => "is public"
+    method_option :is_base, :type => :boolean, :aliases => ["-b", "--b"], :default => false, :desc => "is base for others images"
+    method_option :message, :type => :string, :aliases => ["-m", "--m"], :default => "", :desc => "commit message for this image"
+
+    def sync_image(docker=false)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      verify_software_installed("docker")
+      is_public = options["is_public"] || false
+      is_base = options["is_base"] || false
+      message = options["message"] || ""
+      image_id = commit_image
+
+      if docker
+        message = "before running experiment"
+        image = is_project_with_docker(Dir.pwd)
+        if image and image.is_docker
+          container = image.get_container
+          if !container
+
+            upload_image(image_id, is_public, is_base, message)
+          else
+            command = ["/bin/bash", "-lc", "cnvrg upload_image #{image_id} #{is_public} #{is_base} #{message}"]
+            puts "Running in contianer"
+            container.exec(command, detach: false)
+          end
+        end
+
+      else
+        upload_image(image_id, is_public, is_base, message)
+      end
+
+    end
+
+    desc 'create_custom_image', 'commit notebook changes to create a new notebook image'
+    method_option :name, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "new image name"
+
+    def create_custom_image()
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir()
+      image_name = options["name"] || ""
+      if !image_name.empty? and image_name == "cnvrg"
+        say "can't create image with the name cnvrg", Thor::Shell::Color::RED
+        exit(1)
+      end
+      begin
+        image = is_project_with_docker(working_dir)
+        if !image or !image.is_docker
+          say "Couldn't find image related to project", Thor::Shell::Color::RED
+          exit(0)
+        end
+
+        image.create_custom_image(image_name)
+        say "Image was updated successfully", Thor::Shell::Color::BLUE
+
+      rescue => e
+        puts e
+      end
+    end
+
+
+    desc 'upload_image', 'commit notebook changes to create a new notebook image'
+
+    def upload_image(image_id, is_public, is_base, *message)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      verify_software_installed("docker")
+      image = Docker::Image.get(image_id)
+      project_home = get_project_home
+      @project = Project.new(project_home)
+      last_local_commit = @project.last_local_commit
+      image_name = @project.slug+"#{last_local_commit}"
+      path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.tar"
+      owner = Cnvrg::CLI.get_owner()
+      if !message.nil? or !message.empty?
+        message = message.join(" ")
+      end
+
+      say "Saving image's current state", Thor::Shell::Color::BLUE
+      image.save(path)
+
+      begin
+        say "Compressing image file to upload", Thor::Shell::Color::BLUE
+        gzipRes = system("gzip -f #{path}")
+        if !gzipRes
+          log_End(1, "can't create gz file from image")
+
+          say "Couldn't create tar file from image", Thor::Shell::Color::RED
+          exit(1)
+        end
+        path = path+".gz"
+        @files = Cnvrg::Files.new(owner, "")
+
+        exit_status = $?.exitstatus
+        if exit_status == 0
+          say "Uploading image file", Thor::Shell::Color::BLUE
+
+          diff =container_changes(Dir.pwd)
+          res = @files.upload_image(path, image_name, owner, is_public, is_base, diff[1], diff[0], diff[2], message, image.commit_id)
+          if res
+            File.delete(path)
+            image_loc = is_project_with_docker(Dir.pwd)
+            image_loc.update_slug(res["result"]["id"])
+
+            checks = Helpers.checkmark()
+            say "#{checks} Done", Thor::Shell::Color::GREEN
+            log_end(0)
+          else
+            say "Couldn't upload image", Thor::Shell::Color::RED
+            log_end(1, "can't create upload imag")
+
+          end
+        else
+          say "Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED
+          log_end(1, "can't create upload imag")
+          exit(1)
+        end
+      rescue => e
+        puts e
+
+      rescue SignalException
+        log_end(-1)
+
+        say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
+        exit(1)
+      end
+    end
+
+    desc '', ''
+
+    def upload_log()
+      log_path = '/home/ds/app/uwsgi.log'
+      loglines = File.new(log_path).readlines
+      logs = loglines.select { |x| x.start_with? "cnvrg_app:" }.collect { |x| x.strip }
+
+    end
+
+    desc '', ''
+
+    def exec_container(container_id, *cmd)
+      container = Docker::Container.get(container_id)
+      container.start()
+      cnvrg_command = cmd.join(" ")
+      command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
+      res = container.exec(command, tty: true,wait:5400)[0]
+      say res
+
+    end
+
+    desc '', ''
+
+    def port_container(container_id)
+      container = Docker::Container.get(container_id)
+      say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
+
+
+    end
+
+    desc '', ''
+
+    def stop_container(container_id)
+      container = Docker::Container.get(container_id)
+      container.stop()
+      container.remove()
+
+    end
+
+    desc '', ''
+    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
+    method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
+    method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/home/ds/run_ipython.sh"
+
+
+    def config_remote(image_name, port=7654)
+      local_images = Docker::Image.all
+
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
+      if docker_image_local.empty?
+        say "no image"
+        exit(1)
+      end
+
+      begin
+        login_content = options["login"]
+        app_dir = options["app_dir"]
+        cmd= options["cmd"]
+
+        image_settings = {
+            'Image' => "#{image_name}:latest",
+            'User' => 'ds',
+            'Cmd' => cmd,
+            'WorkingDir' => app_dir,
+            'ExposedPorts' => {
+                '8888/tcp' => {},
+            },
+            'HostConfig' => {
+                'PortBindings' => {
+                    '8888/tcp' => [
+                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
+                    ],
+                },
+            },
+        }
+        container = Docker::Container.create(image_settings)
+        container.start()
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        say "#{container.id}:#{port}"
+      rescue => e
+        if e.message.include? "is not running"
+          return config_remote(image_name, port-1)
+        end
+        puts "error"
+        puts e
+        if container
+          container.kill()
+        end
+        return false
+      end
+    end
+
+    desc '', ''
+    method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
+
+    def config_flask_remote(image_name, port=80)
+      local_images = Docker::Image.all
+
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
+      if docker_image_local.empty?
+        say "no image"
+        exit(1)
+      end
+
+      begin
+        login_content = options["login"]
+        image_settings = {
+            'Image' => "#{image_name}:latest",
+            'User' => 'ds',
+            'Cmd' => '/home/ds/start_super.sh',
+            'WorkingDir' => '/home/ds/app',
+            'ExposedPorts' => {
+                '80/tcp' => {},
+            },
+            'HostConfig' => {
+                'PortBindings' => {
+                    '80/tcp' => [
+                        {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
+                    ],
+                },
+            },
+        }
+        container = Docker::Container.create(image_settings)
+        container.start()
+        command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
+        container.exec(command, tty: true)
+        say "#{container.id}:#{port}"
+      rescue => e
+        if e.message.include? "is not running"
+          return "port is taken"
+        end
+        puts "error"
+        puts e
+        if container
+          container.kill()
+        end
+        return false
+      end
+    end
+
+    desc '', ''
+
+    def upload_cnvrg_image(image_name)
+      verify_logged_in(false)
+      log_start(__method__, args, options)
+      verify_software_installed("docker")
+      owner = Cnvrg::CLI.get_owner()
+
+      path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
+      @files = Cnvrg::Files.new("", "")
+      python_arr = Cnvrg::Images.get_installed_packages("python")
+      py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
+      system_arr = Cnvrg::Images.get_installed_packages("system")
+      sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
+      bash_history = Cnvrg::Images.get_bash_history
+      res = @files.upload_image(path, image_name, owner, true, true, sys, py, bash_history, "Image made by cnvrg.io team", "")
+
+    end
+
+
+    desc '', ''
+
+    def download_built_image(image_name, image_slug)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        owner = Cnvrg::CLI.get_owner()
+        path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.tar.gz"
+        @files = Cnvrg::Files.new(owner, "")
+
+        say "Downloading image file", Thor::Shell::Color::BLUE
+        begin
+          if @files.download_image(path, image_slug, owner)
+            gzipRes = system("gunzip -f #{path}")
+            if !gzipRes
+              log_End(1, "can't unzip gz file from image")
+
+              say "Couldn't create tar file from image", Thor::Shell::Color::RED
+              exit(1)
+            else
+              path = path.gsub(".gz", "")
+              return path
+            end
+
+          else
+            say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
+            log_end(1, "can't download image")
+            return false
+          end
+        rescue Interrupt
+          log_end(-1)
+          say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
+          exit(1)
+        end
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      end
+    end
+
+    desc '', ''
+
+    def download_image(image_name, image_slug)
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        owner = Cnvrg::CLI.get_owner()
+        path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
+        @files = Cnvrg::Files.new(owner, "")
+
+        say "Downloading image file", Thor::Shell::Color::BLUE
+        begin
+          if @files.download_image(path, image_slug, owner)
+
+            dir_path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}"
+            FileUtils.mkdir_p([dir_path])
+            Zip::File.open(path) do |zip_file|
+              # Handle entries one by one
+              zip_file.each do |entry|
+                f_path=File.join(dir_path, entry.name)
+                zip_file.extract(entry, f_path)
+              end
+            end
+
+            return dir_path
+
+          else
+            say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
+            log_end(1, "can't download image")
+            return false
+          end
+        rescue Interrupt
+          log_end(-1)
+          say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
+          exit(1)
+        end
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      ensure
+        if !path.nil?
+          FileUtils.rm(path)
+        end
+      end
+    end
+
+
+    desc 'list_images', 'lists all custom images you can pull'
+
+    def list_images
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        owner = Cnvrg::CLI.get_owner()
+        res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
+        if Cnvrg::CLI.is_response_success(res)
+          printf "%-20s %-20s  %-30s %-20s %-20s %-20s\n", "name", "project", "message", "created by", "is_public", "last updated"
+          res["result"]["images"].each do |u|
+            time = Time.parse(u["created_at"])
+            update_at = get_local_time(time)
+            created_by = u["created_by"]
+
+            printf "%-20s %-20s  %-30s %-20s %-20s %-20s\n", u["name"], u["project"], u["message"], created_by, u["is_public"], update_at
+          end
+        end
+        log_end(0)
+        return res["result"]["images"]
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
+      end
+
+
+    end
+
+    desc 'list_machines', 'lists all machines belong to your organization'
+
+    def list_machines
+      begin
+        verify_logged_in(false)
+        log_start(__method__, args, options)
+        owner = Cnvrg::CLI.get_owner()
+        res = Cnvrg::API.request("users/#{owner}/machines/list", 'GET')
+        if Cnvrg::CLI.is_response_success(res)
+          printf "%-20s %-20s %-20s\n", "name", "created by", "last_used"
+          res["result"]["machines"].each do |u|
+            time = Time.parse(u["last_used"])
+            update_at = get_local_time(time)
+            printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at
+          end
+        end
+        log_end(0)
+        return res["result"]["images"]
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
       end
 
-      desc 'commit_notebook', 'commit notebook changes to create a new notebook image'
 
-      def commit_notebook(notebook_image_name)
-        verify_logged_in(false)
-        log_start(__method__,args,options)
-        begin
-          docker_path = verify_software_installed("docker")
-
-          container_id = get_container_id
-          owner = Cnvrg::CLI.get_owner()
-          say "Commiting notebook changes to: #{notebook_image_name}", Thor::Shell::Color::BLUE
+    end
 
-          commit_res = system("#{docker_path} commit #{container_id} #{notebook_image_name}")
-          if commit_res
-            checker = Helpers.checkmark()
-            log_end(0)
-            say "#{checker} Done.", Thor::Shell::Color::GREEN
-          else
-            log_End(1,"can't commit new notebook image")
-            say "Couldn't commit new notebook image ", Thor::Shell::Color::RED
+    desc 'get_machine', 'create new aws machine'
 
+    def get_machine()
+      begin
+        verify_logged_in(true)
+        log_start(__method__, args, options)
+        owner = Cnvrg::CLI.get_owner()
+        working_dir = is_cnvrg_dir
+        @image = Images.new(working_dir)
+        if @image.nil? or !@image.is_docker
+          say "Couldn't find image related to this project", Thor::Shell::Color::RED
+          exit(0)
+        end
+        res = Cnvrg::API.request("users/#{owner}/machines/list", 'GET')
+        if Cnvrg::CLI.is_response_success(res)
+          if res["result"]["machines"].empty?
+            create = yes? "No machines available, create new machine?", Thor::Shell::Color::YELLOW
+            if create
+              instance_type = machine_options(res["result"]["aws_options"])
+            else
+              exit(0)
+            end
+          end
+          printf "%-20s %-20s %-20s\n", "name", "created by", "last_used", "instance_type"
+          res["result"]["machines"].each do |u|
+            time = Time.parse(u["last_used"])
+            update_at = get_local_time(time)
+            printf "%-20s %-20s %-20s\n", u["name"], u["created_by"], update_at, u["instance type"]
           end
-        rescue SignalException
-          log_end(-1)
-          say "/nAborting"
-          exit(1)
         end
+
+      rescue SignalException
+        log_end(-1)
+        say "\nAborting"
+        exit(1)
       end
+    end
+
 
-      desc 'upload_notebook', 'commit notebook changes to create a new notebook image'
+    desc 'pull_image', 'downloads and loads an image'
 
-      def upload_image(image_name)
+    def pull_image(image_name)
+      begin
         verify_logged_in(false)
-        log_start(__method__,args,options)
-        docker_path = verify_software_installed("docker")
+        log_start(__method__, args, options)
         owner = Cnvrg::CLI.get_owner()
-        # verify image exist
-        images = `#{docker_path} images |grep #{image_name}`
-        if images.empty?
-          say "Couldn't find any images named: #{image_name}", Thor::Shell::Color::RED
+        image = Cnvrg::Images.image_exist(owner, image_name)
+        if !image
+          say "Couldn't find image in cnvrg repository", Thor::Shell::Color::RED
           exit(1)
         end
-        path = File.expand_path('~')+"/.cnvrg/#{owner}_#{image_name}.tar"
-        begin
-          say "Creating image file to upload", Thor::Shell::Color::BLUE
-          if !(File.exist? path or File.exist? path+"gz")
-            saveRes = system("#{docker_path} save #{image_name}>#{path}")
-            if !saveRes
-              log_End(1, "can't create tar file from image")
-              say "Couldn't create tar file from image", Thor::Shell::Color::RED
-              exit(1)
-            end
-            gzipRes = system("gzip -f #{path}")
-            if !gzipRes
-              log_End(1, "can't create tar file from image")
-
-              say "Couldn't create tar file from image", Thor::Shell::Color::RED
-              exit(1)
-            end
-          end
-
-          path = path+".gz"
-          @files = Cnvrg::Files.new(owner, "")
-
-          exit_status = $?.exitstatus
-          if exit_status == 0
-            say "Uploading image file", Thor::Shell::Color::BLUE
-            res = @files.upload_image(path, image_name, owner)
-            if res
-              File.delete(path)
-              checks = Helpers.checkmark()
-              say "#{checks} Done", Thor::Shell::Color::GREEN
-              log_end(0)
-            else
-              say "Couldn't upload image", Thor::Shell::Color::RED
-              log_end(1, "can't create upload imag")
-
-            end
+        path = download_image(image_name, image["slug"])
+        if path
+          say "Building image", Thor::Shell::Color::BLUE
+          Docker.options[:read_timeout]=1200
+          image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile', 't' => "#{image_name}:lastest"})
+          if not image.nil?
+            say "Finished downloading image, cleaning up..", Thor::Shell::Color::GREEN
+            FileUtils.rm_rf(path)
+            checks = Helpers.checkmark()
+            say "#{checks} Done", Thor::Shell::Color::GREEN
+            log_end(0)
+            return image
+            log_end(0)
           else
-            say "Couldn't create image file for: #{image_name}", Thor::Shell::Color::RED
-            log_end(1, "can't create upload imag")
-            exit(1)
+            say "Could not build image", Thor::Shell::Color::RED
+            log_end(1, "Could build  image")
+            return false
           end
-        rescue SignalException
-          log_end(-1)
+        else
+          say "Could not download image", Thor::Shell::Color::RED
+          log_end(1, "Could build  image")
+          return false
 
-          say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
-          exit(1)
-        end
 
+        end
 
-      end
+          # else
+          #   path = download_image(image_name,image["slug"])
+          #   if path
+          #     image = Docker::Image.import(path)
+          #     image.tag('repo' => image_name, 'tag' => 'latest')
+          #     if not image.nil?
+          #       say "Finished downloading image, cleaning up..", Thor::Shell::Color::GREEN
+          #       FileUtils.rm(path)
+          #       checks = Helpers.checkmark()
+          #       say "#{checks} Done", Thor::Shell::Color::GREEN
+          #       log_end(0)
+          #       return image
+          #       log_end(0)
+          #     else
+          #       say "Could not download image", Thor::Shell::Color::RED
+          #       return false
+          #     end
+          #
+          #   end
+          # end
+      rescue
+        say "Couldn't build image", Thor::Shell::Color::RED
 
-      desc 'download_image', 'commit notebook changes to create a new notebook image'
+      rescue SignalException
+        say "\nAborting"
+        log_end(-1)
+        exit(1)
+      ensure
+        if path
+          FileUtils.rm_rf(path)
 
-      def download_image(image_name)
-        begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
-          owner = Cnvrg::CLI.get_owner()
+        end
+      end
 
-          notebooks_res = Cnvrg::API.request("users/#{owner}/images/" + "find", 'POST', {image_name: image_name})
 
-          if Cnvrg::CLI.is_response_success(notebooks_res)
+    end
 
-            images = notebooks_res["result"]["images"]
-            if images.empty?
-              say "Couldn't find any image with name: #{image_name}", Thor::Shell::Color::RED
-              exit(1)
-            elsif images.size == 1
-              image_id = images[0]["slug"]
-            else
-              printf "%-20s %-20s %-30s %-20s\n", "name", "version", "last updated", "created by"
-              images.each_with_index do |u, i|
-                time = Time.parse(u["updated_at"])
-                update_at = get_local_time(time)
-                version = u["version"] || "v1"
-                created_by = u["image_file_name"][/^[^\_]*/]
-                printf "%-20s %-20s %-30s %-20s\n", u["name"], version, update_at, created_by
-              end
-              choice = ask("Which version to download for #{image_name}?")
-              images.each do |u|
-                if u["version"] == choice
-                  image_id = u["slug"]
-                end
+    desc 'set_image', 'set image to a porject'
 
-              end
+    def set_image(docker_image)
+      verify_logged_in(true)
+      log_start(__method__, args, options)
+      working_dir = is_cnvrg_dir
+      project = Project.new(working_dir)
 
-            end
-          end
-          path = Dir.pwd+"/#{image_name}.tar.gz"
-          @files = Cnvrg::Files.new(owner, "")
+      local_images = Docker::Image.all
+      docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.include? docker_image }.flatten
+      if docker_image_local.size == 0
 
-          say "Downloading image file", Thor::Shell::Color::BLUE
-          begin
-            image_path = @files.download_image(path, image_id, owner)
-            return image_path
-            # if res
-            #   checks = Helpers.checkmark()
-            #   say "#{checks} Done", Thor::Shell::Color::GREEN
-            #   log_end(0)
-            #   return true
-            # else
-            #   say "Couldn't download image #{image_name}", Thor::Shell::Color::RED
-            #   log_end(1,"can't download image")
-            #   return false
-            # end
-          rescue Interrupt
-            log_end(-1)
-            say "The user has exited to process, aborting", Thor::Shell::Color::BLUE
+        if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
+          image = pull_image(docker_image)
+          if image
+            say "downloaded image: #{docker_image}"
+            @image = Images.new(working_dir, docker_image)
+          else
+            say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
             exit(1)
           end
-        rescue SignalException
-          log_end(-1)
-          say "/nAborting"
+        else
+          say "Could not create a new project with docker, image was not found", Thor::Shell::Color::RED
           exit(1)
+
         end
+      elsif docker_image_local.size == 1
+        say "found image: #{docker_image_local[0]}, setting it up..", Thor::Shell::Color::BLUE
+        @image = Images.new(working_dir, docker_image_local[0])
+      elsif docker_image_local.size >1
+        say "found #{docker_image_local.size} images, choose the image name you want to use", Thor::Shell::Color::BLUE
+        image_name = ask "#{docker_image_local.join("\n")}\n", Thor::Shell::Color::BLUE
+        image_name = image_name.strip
+        @image = Images.new(working_dir, image_name)
       end
+      @image.update_image_activity(project.last_local_commit, nil)
+    end
 
 
-      desc 'list_images', 'lists all custom images you can pull'
+    no_tasks do
+      def get_instance_type(instances)
+        machines = instances.map { |x| x }.select { |y| y[1] }
+        if machines.size == 1
+          instance_type = machines[0][0]
+        elsif machines.size > 1
 
-      def list_images
-        begin
-        verify_logged_in(false)
-        log_start(__method__,args,options)
-        owner = Cnvrg::CLI.get_owner()
-        res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
-        if Cnvrg::CLI.is_response_success(res)
-          printf "%-20s %-20s %-30s %-20s\n", "name", "version", "last updated", "created by"
-          res["result"]["images"].each do |u|
-            time = Time.parse(u["updated_at"])
-            update_at = get_local_time(time)
-            version = u["version"] || "v1"
-            created_by = u["image_file_name"][/^[^\_]*/]
+          values = instances.map { |x| x[0] }
+          machines_str = "#{machines.map { |x| " " +x[0] }.flatten.join(" or")}"
 
-            printf "%-20s %-20s %-30s %-20s\n", u["name"], version, update_at, created_by
+          instance_type = ask ("You can't choose more than 1 machine,either:#{machines_str}")
+          instance_type = values.select { |x| x.eql? instance_type }
+          if instance_type.empty?
+            instance_type = nil
+          else
+            instance_type = instance_type[0]
           end
+        else
+          instance_type = nil
         end
-        log_end(0)
-        return res["result"]["images"]
-        rescue SignalException
-          log_end(-1)
-          say "/nAborting"
-          exit(1)
-        end
+        return instance_type
+      end
 
+      def get_image(dir)
+        project_dir = is_cnvrg_dir(dir)
+        if !project_dir
+          return false
+        else
+          project_config = YAML.load_file(project_dir+"/.cnvrg/config.yml")
+          if project_config.to_h[:docker]
+            project_config.to_h[:image_base]
+          else
+            false
+          end
+        end
 
       end
 
+      def set_owner(owner, username, url)
+        home_dir = File.expand_path('~')
 
-      desc 'pull_image', 'downloads and loads a custom image'
-      def pull_image(image_name)
         begin
-          verify_logged_in(false)
-          log_start(__method__,args,options)
-
-          path = download_image(image_name)
-          if path
-            image = Docker::Image.import(path)
-            if not image.nil?
-              return image
-              log_end(0)
-            else
-              say "Could not download image", Thor::Shell::Color::RED
-              log_end(1,loadRes)
-              retunr false
-            end
-
+          if !File.directory? home_dir+"/.cnvrg"
+            FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
           end
-        rescue SignalException
-          say "\nAborting"
-          log_end(-1)
-          exit(1)
+          if !File.exist?(home_dir+"/.cnvrg/config.yml")
+            FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
+          end
+          if url.nil? or url.empty?
+            url = "https://cnvrg.io/api"
+          end
+          config = {owner: owner, username: username, version_last_check: get_start_day(), api: url}
+
+          File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          return true
+        rescue
+          return false
         end
 
       end
 
-      no_tasks do
-        def get_container_id
-          docker_path = verify_software_installed("docker")
+      def get_start_day
+        time = Time.now
+        return Time.new(time.year, time.month, time.day)
+      end
 
-          container_id = `#{docker_path} ps --quiet --last=1  --filter status=running`.strip!
-          if container_id.empty?
-            say "Couldn't find a running notebook server, execute run_notebook", Thor::Shell::Color::RED
-            exit(1)
-          end
-          return container_id
+      def calc_output_time(upload_output)
+        if upload_output.nil? or upload_output.empty?
+          return 0
+        end
+        time = upload_output.split(/(\d+)/).reject(&:empty?).map { |x| x.strip }
+        if time.size!=2
+          upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\nre-enter value for upload_output")
+          return calc_output_time(upload_output)
+        end
+        if time[0] == "0"
+          return -1
+        end
+        case time[1].downcase
+          when "s"
+            return time[0].to_f
+          when "m"
+            return time[0].to_f*60
+          when "h"
+            return time[0].to_f*3600
+          when "d"
+            return time[0].to_f*24*3600
+          else
+            upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\n re-enter value for upload_output")
+            calc_output_time(upload_output)
         end
 
-        def set_owner(owner, username)
-          home_dir = File.expand_path('~')
-
-          begin
-            if !File.directory? home_dir+"/.cnvrg"
-              FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp"])
-            end
-            if !File.exist?(home_dir+"/.cnvrg/config.yml")
-              FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
-            end
-            config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
-            if !config or config.empty?
-              config = {owner: owner, username: username, version_last_check: get_start_day(), api: "https://cnvrg.io/api"}
-            else
-              config = {owner: owner, username: username, version_last_check: get_start_day(), api: config.to_h[:api]}
-            end
-
-            File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-            return true
-          rescue
-            return false
-          end
+      end
 
+      def log_start(command, args="", options={})
+        begin
+          $LOG.info "----"
+          $LOG.info ruby_version: RUBY_VERSION, os: Cnvrg::Helpers.os(), cli_version: Cnvrg::VERSION
+          $LOG.info command: command, args: args, options: options
+        rescue
         end
+      end
 
-        def get_start_day
-          time = Time.now
-          return Time.new(time.year, time.month, time.day)
+      def log_end(exit_status=0, error=nil)
+        begin
+          if exit_status==1
+            $LOG.error exit_status: exit_status, error: error
+          else
+            $LOG.info exit_status: exit_status
+          end
+        rescue
         end
+      end
 
-        def calc_output_time(upload_output)
-          if upload_output.nil? or upload_output.empty?
-            return 0
-          end
-          time = upload_output.split(/(\d+)/).reject(&:empty?).map { |x| x.strip }
-          if time.size!=2
-            upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\nre-enter value for upload_output")
-            return calc_output_time(upload_output)
-          end
-          if time[0] == "0"
-            return -1
+      def self.is_response_success(response, should_exit=true)
+        if response.nil?
+          if !Cnvrg::Helpers.internet_connection?
+            # Cnvrg::CLI.log_end(1,"no internet connection")
+            say("<%= color('Error:You seems to be offline', RED) %>")
+          else
+            say("<%= color('Error', RED) %>")
           end
-          case time[1].downcase
-            when "s"
-              return time[0].to_f
-            when "m"
-              return time[0].to_f*60
-            when "h"
-              return time[0].to_f*3600
-            when "d"
-              return time[0].to_f*24*3600
-            else
-              upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\n re-enter value for upload_output")
-              calc_output_time(upload_output)
+        elsif response["status"]!= 200
+          error = response['message']
+          # Cnvrg::CLI.log_end(1, error)
+          if response["status"] == 500
+            say("<%= color('Server Error', RED) %>")
+          else
+            say("<%= color('Error:#{error}', RED) %>")
           end
 
-        end
-        def log_start(command,args="", options={})
-          begin
-            $LOG.info "----"
-            $LOG.info ruby_version: RUBY_VERSION, os: Cnvrg::Helpers.os(), cli_version:Cnvrg::VERSION
-            $LOG.info command:command,args:args,options:options
-          rescue
-          end
-        end
-        def log_end(exit_status=0, error=nil)
-          begin
-            if exit_status==1
-              $LOG.error exit_status:exit_status, error:error
-            else
-              $LOG.info exit_status:exit_status
-            end
-          rescue
+          if should_exit
+            exit(1)
+          else
+            return false
           end
         end
+        return true
+      end
 
-        def self.is_response_success(response, should_exit=true)
-          if response.nil?
-            if !Cnvrg::Helpers.internet_connection?
-              # Cnvrg::CLI.log_end(1,"no internet connection")
-                say("<%= color('Error:You seems to be offline', RED) %>")
-            else
-              say("<%= color('Error', RED) %>")
-            end
-          elsif response["status"]!= 200
-            error = response['message']
-            # Cnvrg::CLI.log_end(1, error)
-            if response["status"] == 500
-              say("<%= color('Server Error', RED) %>")
-            else
-              say("<%= color('Error:#{error}', RED) %>")
-            end
+      def self.get_owner
+        home_dir = File.expand_path('~')
 
-            if should_exit
-              exit(1)
-            else
-              return false
-            end
-          end
-          return true
+        config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
+        owner = config.to_h[:owner]
+        if owner.empty?
+          invoke :set_default_owner, [], []
+          return get_owner()
+        else
+          return owner
         end
+      end
+      def get_base_url
+        home_dir = File.expand_path('~')
 
-        def self.get_owner
-          home_dir = File.expand_path('~')
-
-          config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
-          owner = config.to_h[:owner]
-          if owner.empty?
-            invoke :set_default_owner
-            return get_owner()
-          else
-            return owner
-          end
-        end
+        config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
+        api = config.to_h[:api]
+        return api.gsub!("/api","")
+      end
 
-        def get_project_home
-          absolute_path = Dir.pwd
-          dirs = absolute_path.split("/")
-          dirs.pop while not Dir.exists?("#{dirs.join("/")}/.cnvrg") and dirs.size != 0
+      def get_project_home
+        absolute_path = Dir.pwd
+        dirs = absolute_path.split("/")
+        dirs.pop while not Dir.exists?("#{dirs.join("/")}/.cnvrg") and dirs.size != 0
 
-          if dirs.size == 0
-            say "Couldn't find cnvrg directory. Please start a new project", Thor::Shell::Color::RED
-            exit(1)
-          end
-          return dirs.join("/")
+        if dirs.size == 0
+          say "Couldn't find cnvrg directory. Please start a new project", Thor::Shell::Color::RED
+          exit(1)
         end
+        return dirs.join("/")
+      end
 
-        def should_update_version
-          res = Cnvrg::API.request("/cli/version", 'GET')
-          if Cnvrg::CLI.is_response_success(res, false)
-            updated_version = res["result"]["version"]
-            if updated_version != Cnvrg::VERSION
-              return true
-            else
-              return false
-            end
+      def should_update_version
+        res = Cnvrg::API.request("/cli/version", 'GET')
+        if Cnvrg::CLI.is_response_success(res, false)
+          updated_version = res["result"]["version"]
+          if updated_version != Cnvrg::VERSION
+            return true
           else
             return false
           end
+        else
+          return false
         end
-        def log_handler
-          begin
+      end
+
+      def log_handler
+        begin
           date = DateTime.now.strftime("%m_%d_%Y")
           logfile = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
           if !File.exist? logfile
@@ -1421,63 +3159,83 @@ module Cnvrg
               count+=1
             end
             @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-            @files.upload_log_file(logfile_old,"log_#{date}.log",yesterday)
+            @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
             FileUtils.remove logfile_old
 
 
           end
-            $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
-          rescue => e
-            # puts e
-          end
+          $LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
+        rescue => e
+          # puts e
         end
+      end
 
 
+      def verify_logged_in(in_dir=true)
+        log_handler()
+        auth = Cnvrg::Auth.new
+        unless auth.is_logged_in?
+          say 'You\'re not logged in', Thor::Shell::Color::RED
+          say 'Please log in via `cnvrg login`', Thor::Shell::Color::BLUE
+          exit(1)
+        end
 
-        def verify_logged_in(in_dir=true)
-          log_handler()
-          auth = Cnvrg::Auth.new
-          unless auth.is_logged_in?
-            say 'You\'re not logged in', Thor::Shell::Color::RED
-            say 'Please log in via `cnvrg login`', Thor::Shell::Color::YELLOW
-            exit(1)
-          end
-
-          if !Helpers.internet_connection?
-              wait_offline = agree "Seems like you're offline, wait until your'  back online?",limited_to: ['y', 'n'], default: 'n'
-              if wait_offline
-                say "Waiting until your'e online..", Thor::Shell::Color::BLUE
-                while !Cnvrg::Helpers.internet_connection?
-                end
-              else
-                say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
-                exit(0)
-              end
+        if !Helpers.internet_connection?
+          wait_offline = agree "Seems like you're offline, wait until your'e back online?", Thor::Shell::Color::YELLOW
+          if wait_offline
+            say "Waiting until your'e online..", Thor::Shell::Color::BLUE
+            while !Cnvrg::Helpers.internet_connection?
+            end
+          else
+            say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
+            exit(0)
           end
+        end
 
-          config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
-          version_date = config.to_h[:version_last_check]
-          next_day = get_start_day+ 86399
-          if not (version_date..next_day).cover?(Time.now)
-            if should_update_version()
-              say "There is a new version, run gem update cnvrg", Thor::Shell::Color::BLUE
-            end
+        config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
+        version_date = config.to_h[:version_last_check]
+        next_day = get_start_day+ 86399
+        if not (version_date..next_day).cover?(Time.now)
+          if should_update_version()
+            say "There is a new version, run gem update cnvrg", Thor::Shell::Color::BLUE
           end
+        end
         if in_dir
-          current_dir = Dir.pwd
-          if not Dir.exist? current_dir+"/.cnvrg"
-            say "You're not in a cnvrg project directory",Thor::Shell::Color::RED
+          is_cnvrg = is_cnvrg_dir
+          if !is_cnvrg
+            say "You're not in a cnvrg project directory", Thor::Shell::Color::RED
             exit(0)
-
           end
+
         end
+        #verify tmp dirs exist
+        home_dir = File.expand_path('~')
+
+        FileUtils.mkdir_p([home_dir+"/.cnvrg", home_dir+"/.cnvrg/tmp", home_dir+"/.cnvrg/tmp_files"])
 
 
+      end
 
+      def is_cnvrg_dir(dir=Dir.pwd)
+        current_dir = dir
+        home_dir = File.expand_path('~')
+        is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
+        until is_cnvrg == true
+          current_dir = File.expand_path("..", current_dir)
+          is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
+          if File.expand_path("..", current_dir).eql? home_dir
+            break
+          end
         end
+        if is_cnvrg
+          return current_dir
+        else
+          return false
+        end
+      end
 
-        def verify_software_installed(software)
-          begin
+      def verify_software_installed(software)
+        begin
           install_url = Cnvrg::CLI::INSTALLATION_URLS[software.to_sym]
           installed = `which #{software}`
           if installed.empty? or installed.nil?
@@ -1496,29 +3254,164 @@ module Cnvrg
                   #TODO: change here
                   say "docker isn't running. run:\ndocker-machine start default 2>/dev/null & eval `docker-machine env default`", Thor::Shell::Color::BLUE
                   exit(1)
-              end
+                end
               end
           end
 
-          return installed
-          rescue
-            exit(1)
+          return true
+        rescue
+          exit(1)
+        end
+
+      end
+
+      def get_local_time(time_to_update)
+        local = Time.now.localtime
+        gmt_offset = local.gmt_offset
+        new_time =time_to_update +gmt_offset
+        return new_time.to_s.gsub("UTC", "")
+
+      end
+
+
+      def is_project_with_docker(dir)
+        project_dir = is_cnvrg_dir(dir)
+        if !project_dir
+          return false
+        else
+          project_config = YAML.load_file(project_dir+"/.cnvrg/config.yml")
+          if project_config.to_h[:docker]
+            image = Images.new(project_dir)
+            return image
+          else
+            false
+          end
+        end
+
+
+      end
+
+      def machine_options(aws_options)
+        begin
+          say "Choose type of machine:", Thor::Shell::Color::BLUE
+          printf "%-20s %-20s %-30s\n", "type", "details", "options"
+          all_options = []
+          aws_options.each do |a|
+            all_options << a["options"].flatten
+            options = a["options"].join(" ")
+            printf "%-20s %-20s %-30s\n", a["type"], a["details"], options
+          end
+          all_options.flatten!
+          instance_type = ask "which type of machine?", Thor::Shell::Color::YELLOW
+          count = 0
+          while !all_options.include? instance_type and count <4
+            say "Couldn't find #{instance_type}", Thor::Shell::Color::RED
+            instance_type = ask "which type of machine?", Thor::Shell::Color::YELLOW
+            count+=1
           end
+          return instance_type
+        rescue
+          return false
+        rescue SignalException
+          say "\nAborting"
+          Exit(0)
+        end
+
 
+      end
+
+      def container_changes(dir)
+        container_id = is_project_with_docker(dir)
+        if not container_id
+          return false
         end
+        container = Docker::Container.get(container_id)
+        command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
+        pip = container.exec(command, tty: true)[0]
+        command = ["/bin/bash", "-lc", "dpkg -l"]
+        dpkg = container.exec(command, tty: true)[0]
+        command = ["/bin/bash", "-lc", "cat /home/ds/.bash_history"]
+        history = container.exec(command, tty: true)[0]
+        diff = [pip, dpkg, history]
+        return diff
+        # File.open(dir+"/.cnvrg/pip_#{date}.txt", "w+") { |f| f.write pip }
+        # File.open(dir+"/.cnvrg/dpkg_#{date}.txt", "w+") { |f| f.write dpkg }
+        # File.open(dir+"/.cnvrg/history_#{date}.txt", "w+") { |f| f.write history }
+        #
+        #   pip_new = pip[0].split("\r\n")
+        #   pip_base_file =JSON.parse(File.open( dir+"/.cnvrg/pip_base.txt","rb").read)
+        #   pip_base = pip_base_file[0].split("\r\n")
+        #
+        #   pip_diff = pip_new - pip_base
+        #   if pip_diff.empty?
+        #     puts "pip is same"
+        #   else
+        #     puts pip_diff
+        #   end
+        #   dpkg_new = dpkg[0].split("\r\n")
+        #   dpkg_base_file =JSON.parse(File.open( dir+"/.cnvrg/dpkg_base.txt","rb").read)
+        #   dpkg_base = dpkg_base_file[0].split("\r\n")
+        #
+        #   dpkg_diff = dpkg_new - dpkg_base
+        #   if dpkg_diff.empty?
+        #     puts "dpkg is same"
+        #   else
+        #     puts dpkg_diff
+        #   end
+        #   puts history
+      end
 
-        def get_local_time(time_to_update)
-          local = Time.now.localtime
-          gmt_offset = local.gmt_offset
-          new_time =time_to_update +gmt_offset
-          return new_time.to_s.gsub("UTC", "")
+      def is_port_taken(ip=Cnvrg::CLI::IP, port=Cnvrg::CLI::PORT, seconds=1)
+        Timeout::timeout(seconds) do
+          begin
+            TCPSocket.new(ip, port).close
+            true
+          rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH
+            false
+          end
+        end
+      rescue Timeout::Error
+        false
+      end
+
+      def image_activity(image)
+        res = image.handle_image_activity()
+        if res == -1
+          #image is not known
+          say "Images #{image.image_name} is not known", Thor::Shell::Color::YELLOW
+          to_sync = yes? "Sync image?", Thor::Shell::Color::YELLOW
+          if to_sync
+            sync_image()
+            res = image.handle_image_activity()
+            return res
+          else
+            return false
+          end
 
         end
+        return res
+
+      end
 
+      def get_note_url(working_dir)
+        config = YAML.load_file(working_dir+"/.cnvrg/config.yml")
+        return config[:notebook_slug]
 
       end
-    end
 
+      def get_schedule_date()
+
+        local = Time.now.localtime
+        # gmt_offset = local.gmt_offset
+        new_time =(local).to_s
+        new_time = new_time[0, new_time.size-6] #remove timezone
+        return new_time
+
+      end
 
 
+    end
+  end
 end
+
+