cnvrg 0.0.11 → 0.0.14.0

data/lib/cnvrg/experiment.rb

@@ -1,35 +1,104 @@
 module Cnvrg
-    class Experiment
-        attr_reader :slug
-        def initialize(owner, project_slug)
-            @project_slug = project_slug
-            @owner = owner
-            @base_resource = "users/#{owner}/projects/#{project_slug}/"
-            @slug = nil
-        end
-
-        def start(input, platform, machine_name, start_commit,name, email_notification)
-            res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
-                                     { input: input, platform: platform, machine_name: machine_name, start_commit: start_commit , title:name, email_notification:email_notification})
-            Cnvrg::CLI.is_response_success(res)
-
-            @slug = res.to_h["result"].to_h["slug"]
-
-            return res
-            
-        end
-
-        def upload_temp_log(temp_log, cpu_average, memory_average )
-            response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', { output: temp_log,
-                                                                                              exp_slug: @slug,cpu_average:cpu_average,
-                                                                                                        memory_average:memory_average})
-            Cnvrg::CLI.is_response_success(response)
-        end
-        def end(output, exit_status, end_commit,cpu_average, memory_average)
-        response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug,
-                                                                                   exit_status: exit_status, end_commit: end_commit,
-                                                                                   cpu_average:cpu_average,memory_average:memory_average })
-         Cnvrg::CLI.is_response_success(response)
-        end
+  class Experiment
+    attr_reader :slug
+
+    def initialize(owner, project_slug)
+      @project_slug = project_slug
+      @owner = owner
+      @base_resource = "users/#{owner}/projects/#{project_slug}/"
+      @slug = nil
+    end
+
+    def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity)
+      res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
+                               {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
+                                title: name, email_notification: email_notification, machine_activity: machine_activity})
+      Cnvrg::CLI.is_response_success(res)
+
+      @slug = res.to_h["result"].to_h["slug"]
+
+      return res
+
+    end
+
+    def start_notebook_session(kernel, start_commit, token, port, remote, notebook_path)
+      res = Cnvrg::API.request(@base_resource + "notebook/start_session", 'POST',
+                               {kernel: kernel, start_commit: start_commit,
+                                 token: token, port: port, remote: remote, notebook_path: notebook_path})
+      Cnvrg::CLI.is_response_success(res)
+
+      @slug = res["result"]["id"]
+
+
+      return @slug
+
+    end
+
+    def end_notebook_session(notebook_slug, end_commit)
+      res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
+                               {notebook_slug: notebook_slug, end_commit: end_commit})
+      Cnvrg::CLI.is_response_success(res)
+
+      return res
+
+    end
+    def update_notebook_slug(proj_dir, slug)
+      begin
+      file = proj_dir+"/.cnvrg/notebook_slug"
+      FileUtils.touch file
+
+      File.open(file, "w+") { |f| f.write slug }
+      rescue
+      end
+
+    end
+
+    def get_notebook_slug(proj_dir)
+      begin
+        notebook_slug = File.open(proj_dir + "/.cnvrg/notebook_slug", "rb").read
+        notebook_slug = notebook_slug.gsub("/n", "")
+        notebook_slug = notebook_slug.to_s.strip
+        return notebook_slug
+      rescue
+        return nil
+      end
+
+
+    end
+    def get_machine_activity(working_dir)
+      begin
+        machine_activity = File.open("/home/ds/notebooks/.cnvrg/machine_activity", "rb").read
+        machine_activity = machine_activity.to_s.strip
+        ma_id = machine_activity.to_i
+        return ma_id
+      rescue
+        return nil
+      end
+
+
+    end
+
+    def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
+      response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
+                                                                                                            commit_sha1: commit_to_run,
+                                                                                                            instance_type: instance_type,
+                                                                                                            scheduling_query:scheduling_query,
+                                                                                                            local_timestamp:local_timestamp})
+      return response
+    end
+
+    def upload_temp_log(temp_log, cpu_average, memory_average)
+      response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
+                                                                                            exp_slug: @slug, cpu_average: cpu_average,
+                                                                                            memory_average: memory_average})
+      Cnvrg::CLI.is_response_success(response)
+    end
+
+    def end(output, exit_status, end_commit, cpu_average, memory_average)
+      response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
+                                                                                exit_status: exit_status, end_commit: end_commit,
+                                                                                cpu_average: cpu_average, memory_average: memory_average})
+      Cnvrg::CLI.is_response_success(response)
     end
+  end
 end

data/lib/cnvrg/files.rb

@@ -54,22 +54,72 @@ module Cnvrg
       return false
 
     end
+    def upload_exec_file(absolute_path,image_name,commit_id)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      content_type = "application/zip"
+      begin
+        upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
+                                                                                                    file_name: file_name,
+                                                                                                    image_name:image_name,
+                                                                                                    file_size: file_size,
+                                                                                                    file_content_type: content_type,
+                                                                                                    project_slug: @project_slug,
+                                                                                                    commit_id:commit_id})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          if upload_resp["result"]["image"] == -1
+            return -1
+          end
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+
+        end
+      if s3_res
+        return upload_resp["result"]["id"]
+      end
+      return false
+    rescue SignalException
 
-    def upload_image(absolute_path, image_name, owner)
+      say "\nAborting"
+      exit(1)
+    end
+
+  end
+
+
+
+  def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
       file_name = File.basename absolute_path
       file_size = File.size(absolute_path).to_f
-      content_type = "application/gzip"
+      if is_base
 
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
       upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload", 'POST_FILE', {relative_path: absolute_path,
-                                                                                   file_name: file_name, image_name: image_name,
-                                                                                   file_size: file_size, file_content_type: content_type})
+                                                                                          file_name: file_name,
+                                                                                          image_name: image_name,
+                                                                                          file_size: file_size,
+                                                                                          file_content_type: content_type,
+                                                                                          is_public: is_public,
+                                                                                          project_slug: @project_slug,
+                                                                                          commit_id:commit_id ,
+                                                                                          dpkg: dpkg,
+                                                                                          libraries: libraries,
+                                                                                          bash_history: bash,
+                                                                                          commit_message:message,
+                                                                                          is_base: is_base})
       # puts upload_resp
       if Cnvrg::CLI.is_response_success(upload_resp, false)
-        s3_res = upload_large_files_s3(upload_resp, absolute_path)
+        path = upload_resp["result"]["path"]
+        s3_res = upload_small_files_s3(path, absolute_path, content_type)
         if s3_res
           commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
           if Cnvrg::CLI.is_response_success(commit_resp, false)
-            return true
+            return commit_resp["result"]["image"]
           else
             return false
           end
@@ -77,6 +127,51 @@ module Cnvrg
         end
       end
       return false
+      rescue =>e
+        puts e
+      end
+
+    end
+    def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
+      file_name = File.basename absolute_path
+      file_size = File.size(absolute_path).to_f
+      if is_base
+
+        content_type = "application/zip"
+      else
+        content_type = "application/gzip"
+      end
+      begin
+        upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
+                                                                                            file_name: file_name,
+                                                                                            image_name: image_name,
+                                                                                            file_size: file_size,
+                                                                                            file_content_type: content_type,
+                                                                                            is_public: is_public,
+                                                                                            dpkg: dpkg,
+                                                                                            libraries: libraries,
+                                                                                            bash_history: bash,
+                                                                                            commit_message:message,
+                                                                                            is_base: is_base})
+        # puts upload_resp
+        if Cnvrg::CLI.is_response_success(upload_resp, false)
+          path = upload_resp["result"]["path"]
+          s3_res = upload_small_files_s3(path, absolute_path, content_type)
+          if s3_res
+            commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
+            if Cnvrg::CLI.is_response_success(commit_resp, false)
+              return commit_resp["result"]["image"]
+            else
+              return false
+            end
+
+          end
+        end
+        return false
+      rescue =>e
+        puts e
+      end
+
     end
 
     def download_image(file_path_to_store, image_slug, owner)
@@ -86,15 +181,18 @@ module Cnvrg
       path = download_resp["result"]["path"]
 
       if Cnvrg::CLI.is_response_success(download_resp, false)
-        # begin
-        #   open(file_path_to_store, 'wb') do |file|
-        #     file << open(path).read
-        #   end
-        # rescue
-        #   return false
-        # end
-
-        return path
+        begin
+          open(file_path_to_store, 'wb') do |file|
+            file << open(path).read
+          end
+
+          return true
+        rescue => e
+          puts e
+          return false
+        end
+
+        return true
       else
         return false
       end
@@ -109,6 +207,7 @@ module Cnvrg
         http_object = Net::HTTP.new(uri.host, uri.port)
         http_object.use_ssl = true if uri.scheme == 'https'
         request = Net::HTTP::Get.new(sts_path)
+
         body = ""
         http_object.start do |http|
           response = http.request request
@@ -175,7 +274,46 @@ module Cnvrg
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
     end
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
+      begin
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
+        Cnvrg::CLI.is_response_success(res, false)
+        if res["result"]
+          download_resp = res
+          filename = download_resp["result"]["filename"]
+
+          absolute_path += ".conflict" if conflict
+          sts_path = download_resp["result"]["path_sts"]
+        uri = URI.parse(sts_path)
+        http_object = Net::HTTP.new(uri.host, uri.port)
+        http_object.use_ssl = true if uri.scheme == 'https'
+        request = Net::HTTP::Get.new(sts_path)
 
+        body = ""
+        http_object.start do |http|
+          response = http.request request
+          body = response.read_body
+        end
+        URLcrypt::key = [body].pack('H*')
+        s3 = Aws::S3::Client.new(
+            :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
+            :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
+            :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
+            :region => URLcrypt.decrypt(download_resp["result"]["region"]))
+        File.open(project_home+"/"+absolute_path, 'wb') do |file|
+          resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
+                                 key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
+        end
+          return true
+        end
+
+      rescue =>e
+        puts e
+        puts e.backtrace
+        return false
+
+      end
+    end
     def download_file(absolute_path, relative_path, project_home, conflict=false)
       res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
       Cnvrg::CLI.is_response_success(res, false)
@@ -216,9 +354,9 @@ module Cnvrg
       end
     end
 
-    def start_commit
+    def start_commit(new_branch)
 
-      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug,
+      response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug,new_branch:new_branch,
                                                                               username: @owner})
       Cnvrg::CLI.is_response_success(response)
       return response

data/lib/cnvrg/helpers.rb

@@ -76,6 +76,8 @@ module Cnvrg
 # for example:
 # some_dir/
 # some_file.txt
+.git/
+.gitignore
             }.strip
     end
 

data/lib/cnvrg/project.rb

@@ -1,238 +1,312 @@
 require 'fileutils'
-
 module Cnvrg
-    class Project
-        attr_reader :slug, :owner, :title, :local_path
+  class Project
+    attr_reader :slug, :owner, :title, :local_path
 
-        RemoteURL ||= "https://cnvrg.io"
+    RemoteURL ||= "https://cnvrg.io"
 
-        def initialize(project_home)
-            config = YAML.load_file(project_home+"/.cnvrg/config.yml")
-            @local_path = project_home
-            @title = config[:project_name]
-            @slug  = config[:project_slug]
-            @owner = config[:owner]
-        end
+    def initialize(project_home)
+      begin
+        config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+        @local_path = project_home
+        @title = config[:project_name]
+        @slug = config[:project_slug]
+        @owner = config[:owner]
+      rescue => e
+      end
+
+    end
+
+
+    def last_local_commit
+      idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
+      return idx[:commit]
+    end
+
+    def url
+      url = Cnvrg::Helpers.remote_url
+      "#{url}/#{self.owner}/projects/#{self.slug}"
+    end
 
+    def update_ignore_list(new_ignore)
+      if new_ignore.empty?
+        return true
+      end
+      begin
+        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+          new_ignore.each do |i|
+            f.puts("#{i}\n")
+          end
+        end
+        return true
+      rescue
+        return false
+      end
+    end
 
-        def last_local_commit
-            idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
-            return idx[:commit]
+    def get_ignore_list
+      ignore_list = []
+      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+        line = line.strip
+        if line.start_with? "#"
+          next
         end
-        
-        def url
-            url = Cnvrg::Helpers.remote_url
-            "#{url}/#{self.owner}/projects/#{self.slug}"
+        if line.end_with? "/"
+          ignore_list << line.chop
+          sub_dirs = Dir.glob("#{line}/**/*").each { |x| x.gsub!("//", "/") }
+          ignore_list << sub_dirs.flatten
+        else
+          ignore_list << line
         end
+      end
+      return ignore_list.flatten
 
-        def update_ignore_list(new_ignore)
-            if new_ignore.empty?
-                return true
-            end
-            begin
-            File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
-                new_ignore.each do |i|
-                    f.puts("#{i}\n")
-                end
-            end
-            return true
-            rescue
-                return false
-            end
+    end
+
+    # Create project
+
+    def self.create(project_name, clean, with_docker=false)
+      if clean
+        list_dirs = [project_name, project_name + "/.cnvrg"]
+      else
+
+        list_dirs = [project_name,
+                     project_name + "/data",
+                     project_name + "/models",
+                     project_name + "/notebooks",
+                     project_name + "/src",
+                     project_name + "/src/visualizations",
+                     project_name + "/src/features",
+                     project_name + "/.cnvrg"
+        ]
+      end
+
+
+      list_files = [
+          project_name + "/README.md",
+          project_name + "/.cnvrgignore",
+          project_name + "/.cnvrg/config.yml"
+      ]
+      cnvrgreadme = Helpers.readme_content
+      cnvrgignore = Helpers.cnvrgignore_content
+
+      begin
+
+        owner = Cnvrg::CLI.get_owner()
+        response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: with_docker})
+        Cnvrg::CLI.is_response_success(response)
+        response = JSON.parse response["result"]
+        project_slug = response["slug"]
+
+        config = {project_name: project_name,
+                  project_slug: project_slug,
+                  owner: owner,
+                  docker: with_docker}
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+
+        File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
+        File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
+      rescue
+        return false
+      end
+      return true
+    end
+
+    def self.link(owner, project_name, path=nil, docker=false)
+        list_dirs = [".cnvrg"
+        ]
+        list_files = [
+            ".cnvrgignore",
+            ".cnvrg/config.yml"
+        ]
+
+        cnvrgreadme = Helpers.readme_content
+        cnvrgignore = Helpers.cnvrgignore_content
+        begin
+          response = Cnvrg::API.request("cli/create_project", 'POST', {title: project_name, owner: owner, is_docker: docker})
+          Cnvrg::CLI.is_response_success(response)
+          response = JSON.parse response["result"]
+          project_slug = response["slug"]
+
+          config = {project_name: project_name,
+                    project_slug: project_slug,
+                    owner: owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
+          if !File.exist? "README" and !File.exist? "README.md"
+            FileUtils.touch [ "README.md" ]
+            File.open("README.md", "w+") { |f| f.write cnvrgreadme }
+          end
+
+        rescue => e
+          puts e
+          return false
         end
-        def get_ignore_list
-            ignore_list = []
-            File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
-                line = line.strip
-                if line.start_with? "#"
-                    next
-                end
-                if line.end_with? "/"
-                    ignore_list << line.chop
-                    sub_dirs = Dir.glob("#{line}/**/*").each{ |x| x.gsub!("//","/")  }
-                    ignore_list << sub_dirs.flatten
-                else
-                    ignore_list << line
-                end
-            end
-          return ignore_list.flatten
+        return true
+      end
+
+      def self.clone_dir(project_slug, project_owner, project_name)
+        list_dirs = [project_name,
+                     project_name + "/.cnvrg"
+        ]
 
+
+        list_files = [
+            project_name + "/.cnvrg/config.yml",
+            project_name+"/.cnvrgignore",
+        ]
+        begin
+          config = {project_name: project_name,
+                    project_slug: project_slug,
+                    owner: project_owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          cnvrgignore = Helpers.cnvrgignore_content
+
+
+          File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(project_name+"/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
+
+        rescue
+          return false
         end
+        return true
+      end
 
-        # Create project
+      def self.clone_dir_remote(project_slug, project_owner, project_name)
+        list_dirs = [
+            ".cnvrg"
+        ]
 
-        def self.create(project_name, path=nil,clean)
-            if clean
-                list_dirs = [project_name, project_name + "/.cnvrg"]
-            else
 
-                list_dirs = [project_name,
-                             project_name + "/data",
-                             project_name + "/models",
-                             project_name + "/notebooks",
-                             project_name + "/src",
-                             project_name + "/src/visualizations",
-                             project_name + "/src/features",
-                             project_name + "/.cnvrg"
-                ]
-            end
+        list_files = [
+            ".cnvrg/config.yml",
+            ".cnvrgignore",
 
+        ]
+        begin
+          config = {project_name: project_name,
+                    project_slug: project_slug,
+                    owner: project_owner}
+          FileUtils.mkdir_p list_dirs
+          FileUtils.touch list_files
+          cnvrgignore = Helpers.cnvrgignore_content
 
-            list_files = [
-                project_name + "/README.md",
-                project_name + "/.cnvrgignore",
-                project_name + "/.cnvrg/config.yml"
-            ]
-            cnvrgreadme = Helpers.readme_content
-            cnvrgignore = Helpers.cnvrgignore_content
-
-            begin
-
-                owner        = Cnvrg::CLI.get_owner()
-                response     = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name, owner:owner })
-                Cnvrg::CLI.is_response_success(response)
-                response     = JSON.parse response["result"]
-                project_slug = response["slug"]
-
-                config = { project_name: project_name,
-                           project_slug: project_slug,
-                           owner: owner }
-                FileUtils.mkdir_p list_dirs
-                FileUtils.touch list_files
-                
-                File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-                File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
-                File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
-            rescue
-                return false
-            end
-            return true
+
+          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
+
+        rescue
+          return false
         end
-        def self.link(project_name, path=nil)
-            list_dirs = [ ".cnvrg"
-            ]
-
-            list_files = [
-                 "README.md",
-                ".cnvrgignore",
-                ".cnvrg/config.yml"
-            ]
-            cnvrgreadme = Helpers.readme_content
-            cnvrgignore = Helpers.cnvrgignore_content
-            begin
-                response     = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name })
-                Cnvrg::CLI.is_response_success(response)
-                response     = JSON.parse response["result"]
-                project_slug = response["slug"]
-                owner        = response["owner"]
-
-                config = { project_name: project_name,
-                           project_slug: project_slug,
-                           owner: owner }
-                FileUtils.mkdir_p list_dirs
-                FileUtils.touch list_files
-                File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-                File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
-                File.open("README.md", "w+") { |f| f.write cnvrgreadme }
-            rescue
-                return false
-            end
-            return true
+        return true
+      end
+
+      def generate_idx
+        if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
+          old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        else
+          old_idx = nil
         end
-        def self.clone_dir(project_slug,project_owner,project_name)
-            list_dirs = [project_name,
-                         project_name + "/.cnvrg"
-            ]
-
-
-            list_files = [
-                project_name + "/.cnvrg/config.yml",
-            ]
-            begin
-                config = { project_name: project_name,
-                           project_slug: project_slug,
-                           owner: project_owner}
-                FileUtils.mkdir_p list_dirs
-                FileUtils.touch list_files
-
-                File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-                File.open(project_name + "/.cnvrg/idx.yml", "w+") { |f| f.write config.to_yaml }
-            rescue
-                return false
+
+        tree_idx = Hash.new(0)
+
+        list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
+        list_ignore = self.get_ignore_list()
+        list.each do |e|
+          label = e.gsub(self.local_path + "/", "")
+          if File.directory? e
+            if list_ignore.include? label
+              next
             end
-            return true
-        end
-        def generate_idx
-            if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-                old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+            tree_idx[label+"/"] = nil
+          else
+            if list_ignore.include? label
+              next
+            end
+            sha1 = Digest::SHA1.file(e).hexdigest
+            if old_idx.nil? or old_idx.to_h[:tree].nil?
+              tree_idx[label] = {sha1: sha1, commit_time: nil}
+            elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
+              tree_idx[label] = {sha1: sha1, commit_time: nil}
             else
-                old_idx = nil
+              tree_idx[label] = old_idx[:tree][label]
             end
+          end
+        end
 
-            tree_idx = Hash.new(0)
-            
-            list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
-            list_ignore = self.get_ignore_list()
-            list.each do |e|
-                label = e.gsub(self.local_path + "/", "")
-                if File.directory? e
-                    if list_ignore.include? label
-                        next
-                    end
-                    tree_idx[label+"/"] = nil
-                else
-                    if list_ignore.include? label
-                        next
-                    end
-                    sha1 = Digest::SHA1.file(e).hexdigest
-                    if old_idx.nil? or old_idx.to_h[:tree].nil?
-                        tree_idx[label] = { sha1: sha1, commit_time: nil }
-                    elsif old_idx[:tree][label].nil? or old_idx[:tree][label][:sha1] != sha1
-                        tree_idx[label] = { sha1: sha1, commit_time: nil }
-                    else
-                        tree_idx[label] = old_idx[:tree][label]
-                    end
-                end
-            end
+        idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
 
-            idx = { commit: old_idx.to_h[:commit], tree: tree_idx }
-            
-            File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
-            return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        end
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+        return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      end
 
-        def get_idx
-            YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        end
-        def clone
-            response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', { project_slug: self.slug})
-            Cnvrg::CLI.is_response_success(response)
-            return response
-        end
-        def compare_idx
-            local_idx = self.generate_idx
-            response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', { idx: local_idx })
-            CLI.is_response_success(response)
-            return response
-        end
-        def update_idx_with_files_commits!(files, commit_time)
+      def get_idx
+        YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      end
 
-            idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-            files.each do |path|
-                idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
-            end
-            File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+      def clone(remote=0, commit)
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/clone", 'POST', {project_slug: self.slug, remote: remote, commit: commit})
+        return response
+      end
+
+      def compare_idx(new_branch, commit=last_local_commit)
+
+        local_idx = self.generate_idx
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
+        CLI.is_response_success(response)
+        return response
+      end
 
-            return true
+      def compare_commit(commit)
+        if commit.nil? or commit.empty?
+          commit = last_local_commit
         end
+        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+        CLI.is_response_success(response)
+        return response["result"]["new_branch"]
+      end
 
-        def update_idx_with_commit!(commit)
-            idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-            idx_hash[:commit] = commit
+      def update_idx_with_files_commits!(files, commit_time)
 
-            File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
-            return true
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        files.each do |path|
+          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
         end
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+
+        return true
+      end
+
+      def deploy(file_to_run, function, input_params, commit_to_run, instance_type, image_slug, scheduling_query, local_timestamp)
+        response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/deploy", 'POST', {file_to_run: file_to_run, function: function,
+                                                                                           image_slug: image_slug, input_params: input_params,
+                                                                                           commit_sha1: commit_to_run,
+                                                                                           instance_type: instance_type,
+                                                                                           scheduling_query: scheduling_query,
+                                                                                           local_timestamp: local_timestamp})
+        return response
+      end
+
+      def update_idx_with_commit!(commit)
+        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        idx_hash[:commit] = commit
+
+        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+        return true
+      end
+
+      def revert(working_dir)
+        FileUtils.rm_rf working_dir
+        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
+        # CLI.is_response_success(response)
+
+
+      end
 
     end
-end
+  end