cnvrg 1.2.7 → 1.3.2

data/lib/cnvrg/dataset.rb

@@ -12,7 +12,7 @@ module Cnvrg
         if project_home.present?
           @local_path = project_home
           @working_dir = project_home
-          config = YAML.load_file(project_home+"/.cnvrg/config.yml")
+          config = YAML.load_file(project_home + "/.cnvrg/config.yml")
           @title = config[:dataset_name]
           @slug = config[:dataset_slug]
           @owner = config[:owner]
@@ -28,10 +28,43 @@ module Cnvrg
     end
 
 
+    def get_storage_client
+      response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/client", 'GET')
+      if Cnvrg::CLI.is_response_success(response, false)
+        client_params = response['client']
+      else
+        client_params = get_storage_client_fallback
+      end
+      Cnvrg::Downloader::Client.factory(client_params)
+    end
+
+    def get_storage_client_fallback
+      response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/download_multi", "POST", {files: []})
+      raise StandardError.new("Can't find dataset credentials") unless Cnvrg::CLI.is_response_success(response, false)
+      files = response['files']
+      storage = files['is_s3'] ? 's3' : 'minio'
+      files['storage'] = storage
+      files
+    end
+
+
+    def get_stats(commit: nil, query: nil)
+      response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/clone", 'POST', {commit: commit, query: query})
+      Cnvrg::CLI.is_response_success(response, true)
+      response['result']
+    end
+
+    def get_clone_chunk(latest_id: nil, chunk_size: 1000, offset: 0, commit: 'latest')
+      response = Cnvrg::API.request("users/#{@owner}/datasets/#{@slug}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id, offset: offset})
+      return nil unless Cnvrg::CLI.is_response_success(response, false)
+      response['result']['files']['keys']
+    end
+
+
     def backup_idx
       Cnvrg::Logger.log_info("Backup idx")
       idx = self.get_idx
-      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') { |f| f.write idx.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml.backup", 'w') {|f| f.write idx.to_yaml}
     end
 
     def restore_idx
@@ -43,17 +76,17 @@ module Cnvrg
 
     def change_url(owner: '', slug: '', title: '')
       config = {dataset_home: title, dataset_slug: slug, owner: owner}
-      File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+      File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
     end
 
 
-    def self.delete(dataset_slug,owner)
+    def self.delete(dataset_slug, owner)
       response = Cnvrg::API.request("users/#{owner}/datasets/#{dataset_slug}/delete", 'DELETE')
       return response
     end
 
     def last_local_commit
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         return nil
       end
       idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
@@ -67,28 +100,31 @@ module Cnvrg
       return response
 
     end
+
     def list(owner)
       response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
       CLI.is_response_success(response)
       return response
 
     end
+
     def search_queries
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/queries/list", 'GET')
       CLI.is_response_success(response)
-      row = [["name","id", "created_at", "username"]]
+      row = [["name", "id", "created_at", "username"]]
       response["results"]["queries"].each do |query|
-        row << [query["name"],query["slug"], query["created_at"].in_time_zone.to_s, query["username"]]
+        row << [query["name"], query["slug"], query["created_at"].in_time_zone.to_s, query["username"]]
       end
       return row
 
     end
+
     def get_query_file(query_slug)
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/search/#{query_slug}", 'GET')
       CLI.is_response_success(response)
-      row = [["Name","Full path","URL"]]
+      row = [["Name", "Full path", "URL"]]
       response["results"]["query_files"].each do |file|
-        row << [file["name"],file["fullpath"],file["s3_url"]]
+        row << [file["name"], file["fullpath"], file["s3_url"]]
       end
       return row
 
@@ -99,7 +135,7 @@ module Cnvrg
       CLI.is_response_success(response)
       begin
         path = self.working_dir
-        File.open("#{path}/#{response["results"]["filename"]}", "w+") { |f| f.write response["results"]["file_content"] }
+        File.open("#{path}/#{response["results"]["filename"]}", "w+") {|f| f.write response["results"]["file_content"]}
         return true
       rescue
         return false
@@ -113,7 +149,7 @@ module Cnvrg
 
     end
 
-    def upload_tags_via_yml(tag_file=nil)
+    def upload_tags_via_yml(tag_file = nil)
       records_yml = YAML.load_file(tag_file)
       tag_file.close
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/data_tags_create", 'POST', {records_yml: records_yml})
@@ -129,7 +165,7 @@ module Cnvrg
       "#{url}/#{self.owner}/projects/#{self.slug}"
     end
 
-    def self.verify_cnvrgignore_exist(dataset_name,remote)
+    def self.verify_cnvrgignore_exist(dataset_name, remote)
       path = ".cnvrgignore"
       if !File.exist? path
         path = "#{dataset_name}/.cnvrgignore"
@@ -142,20 +178,21 @@ module Cnvrg
           ]
           FileUtils.touch list_files
           cnvrgignore = Helpers.cnvrgignore_content
-          File.open(path, "w+") { |f| f.write cnvrgignore }
+          File.open(path, "w+") {|f| f.write cnvrgignore}
         rescue => e
           return false
         end
 
       end
     end
+
     def update_ignore_list(new_ignore)
 
       if new_ignore.nil? or new_ignore.empty?
         return true
       end
       begin
-        File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
+        File.open(self.local_path + "/.cnvrgignore", "a+") do |f|
           f.puts("\n")
 
           new_ignore.each do |i|
@@ -169,8 +206,12 @@ module Cnvrg
     end
 
     def get_ignore_list
+      ### handle case when after clone .cnvrgignore doesnt exists
+      if not File.exists?(self.local_path + "/.cnvrgignore")
+        self.generate_cnvrg_ignore
+      end
       ignore_list = []
-      File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+      File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
         line = line.strip
         if line.start_with? "#" or ignore_list.include? line or line.empty?
           next
@@ -192,38 +233,38 @@ module Cnvrg
     end
 
 
-    def self.init(owner, dataset_name, is_public=false)
-        list_dirs = [".cnvrg"
-        ]
-        list_files = [
-            ".cnvrg/config.yml"
-        ]
-        create_ignore = false
-        if !File.exist? ".cnvrgignore"
-          list_files << ".cnvrgignore"
-          create_ignore  = true
-        end
+    def self.init(owner, dataset_name, is_public = false, bucket: nil)
+      list_dirs = [".cnvrg"
+      ]
+      list_files = [
+          ".cnvrg/config.yml"
+      ]
+      create_ignore = false
+      if !File.exist? ".cnvrgignore"
+        list_files << ".cnvrgignore"
+        create_ignore = true
+      end
 
 
-        cnvrgignore = Helpers.cnvrgignore_content
-        begin
-            response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
-            Cnvrg::CLI.is_response_success(response)
-          response = JSON.parse response["result"]
-          dataset_slug = response["slug"]
+      cnvrgignore = Helpers.cnvrgignore_content
+      begin
+        response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public, bucket: bucket})
+        Cnvrg::CLI.is_response_success(response)
+        response = JSON.parse response["result"]
+        dataset_slug = response["slug"]
 
-          config = {dataset_name: dataset_name,
-                    dataset_slug: dataset_slug,
-                    owner: owner}
+        config = {dataset_name: dataset_name,
+                  dataset_slug: dataset_slug,
+                  owner: owner}
 
-          FileUtils.mkdir_p list_dirs
-          FileUtils.touch list_files
-          File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-          File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless !create_ignore
-        rescue => e
-          return false
-        end
-        return true
+        FileUtils.mkdir_p list_dirs
+        FileUtils.touch list_files
+        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+        File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore
+      rescue => e
+        return false
+      end
+      return true
     end
 
 
@@ -236,7 +277,7 @@ module Cnvrg
       create_ignore = false
       if !File.exist? ".cnvrgignore"
         list_files << ".cnvrgignore"
-        create_ignore  = true
+        create_ignore = true
       end
 
 
@@ -249,15 +290,20 @@ module Cnvrg
 
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
-        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
-        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless !create_ignore
+        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
+        File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless !create_ignore
       rescue => e
         return false
       end
       return true
     end
 
-    def self.verify_datasets(dataset_titles, timeout=100)
+    def generate_cnvrg_ignore
+      cnvrgignore = Helpers.cnvrgignore_content
+      File.open(self.local_path + "/.cnvrgignore", "w+") {|f| f.write cnvrgignore}
+    end
+
+    def self.verify_datasets(dataset_titles, timeout = 100)
       start_time = Time.now.to_i
       Cnvrg::Logger.log_info("Verifying datasets #{dataset_titles}")
       Cnvrg::Logger.log_info("Timeout is  #{timeout}")
@@ -280,18 +326,18 @@ module Cnvrg
       end
     end
 
-    def clone( commit)
+    def clone(commit)
 
 
       return response
     end
 
-    def self.clone(owner, dataset_name, dataset_slug, remote=false)
+    def self.clone(owner, dataset_name, dataset_slug, remote = false)
 
       begin
         list_dirs = []
         if !remote
-          list_dirs <<  dataset_name
+          list_dirs << dataset_name
         end
         list_dirs << "#{dataset_name}/.cnvrg"
         list_files = [
@@ -305,7 +351,7 @@ module Cnvrg
 
         FileUtils.mkdir_p list_dirs
         FileUtils.touch list_files
-        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open("#{dataset_name}/.cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
       rescue => e
         puts "Exception in clone request:#{e.message}"
         return false
@@ -325,11 +371,11 @@ module Cnvrg
       trees = @files.get_trees(commit: commit)
       return false if trees.nil?
       pb = ProgressBar.create(:title => "Download Progress",
-                         :progress_mark => '=',
-                         :format => "%b>>%i| %p%% %t",
-                         :starting_at => 0,
-                         :total => trees.size,
-                         :autofinish => true)
+                              :progress_mark => '=',
+                              :format => "%b>>%i| %p%% %t",
+                              :starting_at => 0,
+                              :total => trees.size,
+                              :autofinish => true)
       trees.each do |tree|
         pb.progress += 1
         @files.download_dir(dataset_home, tree)
@@ -343,16 +389,16 @@ module Cnvrg
       file_path = ".cnvrg/config.yml"
       file_path = "#{@slug}/" + file_path if !in_folder
       if File.exist?(file_path)
-        File.open(file_path, "a") { |f| f.puts(":success: true") }
+        File.open(file_path, "a") {|f| f.puts(":success: true")}
       end
     rescue
     end
 
-    def self.init_container(owner, dataset_slug,dataset_name)
+    def self.init_container(owner, dataset_slug, dataset_name)
 
       cnvrgignore = Helpers.cnvrgignore_content
       begin
-        list_dirs = [ ".cnvrg"
+        list_dirs = [".cnvrg"
         ]
         list_files = [
             ".cnvrgignore",
@@ -364,9 +410,9 @@ module Cnvrg
         config = {dataset_name: dataset_name,
                   dataset_slug: dataset_slug,
                   owner: owner}
-        File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
+        File.open(".cnvrg/config.yml", "w+") {|f| f.write config.to_yaml}
 
-        File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
+        File.open(".cnvrgignore", "w+") {|f| f.write cnvrgignore} unless File.exist? ".cnvrgignore"
       rescue => e
         return false
       end
@@ -383,23 +429,24 @@ module Cnvrg
     end
 
     def set_idx(idx)
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w+') { |f| f.write idx.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w+') {|f| f.write idx.to_yaml}
     end
 
     def url
       url = Cnvrg::Helpers.remote_url
       "#{url}/#{self.owner}/datasets/#{self.slug}"
     end
-    def generate_chunked_idx(list_files=[], threads: IDXParallelThreads, prefix: '')
+
+    def generate_chunked_idx(list_files = [], threads: IDXParallelThreads, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
         #check if prefix exists do prefix/path otherwise path
         label = file.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if File.directory? file
-          tree[label+"/"] = nil
+          tree[label + "/"] = nil
         else
-          sha1 =  OpenSSL::Digest::SHA1.file(file).hexdigest
+          sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
           file_name = File.basename file
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)
@@ -425,30 +472,30 @@ module Cnvrg
       if commit.blank?
         resp = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/last_valid_commit", 'GET')
         if CLI.is_response_success(resp, false)
-         commit = resp['result']['commit_sha1']
+          commit = resp['result']['commit_sha1']
         end
       end
       self.update_idx_with_commit(commit) if commit.present?
       self.revert_next_commit
     end
 
-    def list_all_files(with_ignore=false)
-      list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
+    def list_all_files(with_ignore = false)
+      list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
 
       #we want that big files will
-      list = list.sort_by{|fn| File.size(fn)}
+      list = list.sort_by {|fn| File.size(fn)}
       return list if with_ignore
-      list_ignore = self.get_ignore_list.map{|ignore_file| "#{self.local_path}/#{ignore_file}"}
+      list_ignore = self.get_ignore_list.map {|ignore_file| "#{self.local_path}/#{ignore_file}"}
       (list - list_ignore)
     end
 
-    def write_idx(tree=nil, commit=nil)
+    def write_idx(tree = nil, commit = nil)
       if tree.blank?
         tree = self.generate_idx[:tree]
-        tree = tree.map{|k,v| (v.present?)? [k, {sha1: v[:sha1], commit_time: Time.now}] : [k,v]}.to_h
+        tree = tree.map {|k, v| (v.present?) ? [k, {sha1: v[:sha1], commit_time: Time.now}] : [k, v]}.to_h
       end
       idx = {tree: tree, commit: commit}
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
     end
 
     def write_tree(tree)
@@ -457,14 +504,14 @@ module Cnvrg
       self.set_idx(idx)
     end
 
-    def generate_idx(show_progress=false)
+    def generate_idx(show_progress = false)
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
         old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       else
         old_idx = nil
       end
       tree_idx = Hash.new(0)
-      list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
+      list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject {|x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~ /^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~ /^#{self.local_path}\/\.cnvrgignore/)}
       list_ignore = self.get_ignore_list()
       if show_progress
         parallel_options = {
@@ -486,18 +533,18 @@ module Cnvrg
         }
       end
 
-      Parallel.map(list, parallel_options ) do |e|
+      Parallel.map(list, parallel_options) do |e|
         label = e.gsub(self.local_path + "/", "")
         if File.directory? e
           if list_ignore.include? label
             next
           end
-          tree_idx[label+"/"] = nil
+          tree_idx[label + "/"] = nil
         else
           if list_ignore.include? label
             next
           end
-          sha1 =  OpenSSL::Digest::SHA1.file(e).hexdigest
+          sha1 = OpenSSL::Digest::SHA1.file(e).hexdigest
           if old_idx.nil? or old_idx.to_h["tree"].nil?
             tree_idx[label] = {sha1: sha1, commit_time: nil}
           elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
@@ -507,15 +554,16 @@ module Cnvrg
           end
         end
       end
-      if !old_idx.nil? and !old_idx[:next_commit].nil? and  !old_idx[:next_commit].empty?
-        idx = {commit: old_idx.to_h[:commit], tree: tree_idx, next_commit:old_idx[:next_commit] }
+      if !old_idx.nil? and !old_idx[:next_commit].nil? and !old_idx[:next_commit].empty?
+        idx = {commit: old_idx.to_h[:commit], tree: tree_idx, next_commit: old_idx[:next_commit]}
       else
         idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
       end
       idx_yaml = idx.to_yaml
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_yaml}
       return idx
     end
+
     def create_volume
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
       CLI.is_response_success(response)
@@ -524,12 +572,13 @@ module Cnvrg
 
     def download_updated_data(current_commit)
       response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_updated_data", 'POST', {current_commit: current_commit})
-      CLI.is_response_success(response,false)
+      CLI.is_response_success(response, false)
       return response
     end
-    def compare_idx(new_branch, commit=last_local_commit,local_idx=nil,force = false, next_commit = nil)
+
+    def compare_idx(new_branch, commit = last_local_commit, local_idx = nil, force = false, next_commit = nil)
       if local_idx.nil?
-        local_idx =  self.generate_idx
+        local_idx = self.generate_idx
       end
       ignore_list = self.get_ignore_list()
       if force
@@ -539,24 +588,24 @@ module Cnvrg
           added.flatten!
         end
 
-        response ={"result"=> {"commit"=>next_commit,"tree"=> {"added"=> added,
-                                                       "updated_on_server"=> [],
-                                                       "updated_on_local"=> [],
-                                                       "deleted"=> [],
-                                                       "conflicts"=> []} } }
+        response = {"result" => {"commit" => next_commit, "tree" => {"added" => added,
+                                                                     "updated_on_server" => [],
+                                                                     "updated_on_local" => [],
+                                                                     "deleted" => [],
+                                                                     "conflicts" => []}}}
         return response
 
       end
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit, ignore:ignore_list, next_commit: next_commit})
-      CLI.is_response_success(response,false)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit, ignore: ignore_list, next_commit: next_commit})
+      CLI.is_response_success(response, false)
       return response
     end
-    
+
     def compare_idx_download(all_files: false, desired_commit: nil)
       current_commit = self.last_local_commit
       next_commit = self.get_next_commit
       ignore_list = self.send_ignore_list()
-      return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, next_commit: next_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
+      return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, next_commit: next_commit, ignore: ignore_list, all_files: all_files, desired_commit: desired_commit.presence})
     end
 
     def set_partial_commit(commit_sha1)
@@ -572,24 +621,25 @@ module Cnvrg
 
 
     def current_status(new_branch)
-        commit=last_local_commit
-        response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})
-        CLI.is_response_success(response,true)
-        return response
+      commit = last_local_commit
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit, new_branch: new_branch})
+      CLI.is_response_success(response, true)
+      return response
     end
+
     def send_ignore_list()
       begin
         ignore_list = []
-        File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
+        File.open(self.local_path + "/.cnvrgignore", "r").each_line do |line|
           line = line.strip
           if line.start_with? "#" or ignore_list.include? line or line.empty?
             next
           end
           if line.end_with? "/"
-            ignore_list << line.gsub("/","")
-            ignore_list << line+"."
+            ignore_list << line.gsub("/", "")
+            ignore_list << line + "."
           elsif line.include? "*"
-            line = line.gsub("*",".*")
+            line = line.gsub("*", ".*")
             ignore_list << line
           else
             ignore_list << line
@@ -603,15 +653,14 @@ module Cnvrg
     end
 
 
-
-
     def compare_commits(commit)
-      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
-      CLI.is_response_success(response,false)
+      response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit, current_commit: last_local_commit})
+      CLI.is_response_success(response, false)
       return response
     end
+
     def set_next_commit(commit_sha1)
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         idx_hash = Hash.new()
         idx_hash[:commit] = ""
         idx_hash[:tree] = ""
@@ -620,37 +669,41 @@ module Cnvrg
 
       end
       idx_hash[:next_commit] = commit_sha1
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
       return true
 
     end
+
     def get_next_commit()
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         return nil
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       return idx_hash[:next_commit]
     end
+
     def remove_next_commit()
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         return nil
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       idx = Hash.new()
       idx[:commit] = idx_hash[:next_commit]
       idx[:tree] = idx_hash[:tree]
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
     end
+
     def revert_next_commit()
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         return nil
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
       idx_hash = idx_hash.except(:next_commit)
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
     end
+
     def get_current_commit()
-      if  !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
+      if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
         return nil
       end
       idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
@@ -658,50 +711,51 @@ module Cnvrg
     end
 
     def compare_commit(commit)
-        if commit.nil? or commit.empty?
-          commit = last_local_commit
-        end
-        response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
-        CLI.is_response_success(response,false)
-        update_is_new_branch(response["result"]["new_branch"])
-        return response["result"]["new_branch"]
+      if commit.nil? or commit.empty?
+        commit = last_local_commit
       end
+      response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
+      CLI.is_response_success(response, false)
+      update_is_new_branch(response["result"]["new_branch"])
+      return response["result"]["new_branch"]
+    end
 
-      def update_idx_with_files_commits!(files, commit_time)
-        # files.flatten!
-        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        # idx_hash[:commit] = commit
-
-        files.each do |path|
-          idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
-        end
-        idx_hash[:next_commit] = idx_hash[:next_commit]
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
+    def update_idx_with_files_commits!(files, commit_time)
+      # files.flatten!
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      # idx_hash[:commit] = commit
 
-        return true
+      files.each do |path|
+        idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
       end
-    def update_idx(idx)
-      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
+      idx_hash[:next_commit] = idx_hash[:next_commit]
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
 
       return true
     end
 
+    def update_idx(idx)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx.to_yaml}
+
+      return true
+    end
 
-      def update_idx_with_commit!(commit)
-        idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
-        idx_hash[:commit] = commit
 
-        File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
-        return true
-      end
+    def update_idx_with_commit!(commit)
+      idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+      idx_hash[:commit] = commit
 
-      def revert(working_dir)
-        FileUtils.rm_rf working_dir
-        # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
-        # CLI.is_response_success(response)
+      File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') {|f| f.write idx_hash.to_yaml}
+      return true
+    end
 
+    def revert(working_dir)
+      FileUtils.rm_rf working_dir
+      # response     = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
+      # CLI.is_response_success(response)
 
-      end
 
     end
+
   end
+end