cnvrg 1.6.38 → 1.9.5

data/lib/cnvrg/connect_job_ssh.rb

@@ -0,0 +1,31 @@
+module Cnvrg
+  class ConnectJobSsh
+    def initialize(job_id)
+      home_dir = File.expand_path('~')
+      config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
+      @owner = config.to_h[:owner]
+      @job_id = job_id
+    rescue => e
+      @owner = ""
+      Cnvrg::Logger.log_info("cnvrg is not configured")
+    end
+
+    def start(username, password)
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/start" , 'POST', {username: username, password: password})
+    end
+
+    def status()
+      Cnvrg::API_V2.request("#{@owner}/job_ssh/#{@job_id}/status" , 'GET', nil)
+    end
+
+    def run_portforward_command(pod_name, port, kubeconfig, namespace)
+      command = "kubectl"
+      if kubeconfig.present?
+        command = "kubectl --kubeconfig=#{kubeconfig}"
+      end
+      bashCommand = "#{command} -n #{namespace} port-forward #{pod_name} #{port}:22"
+      puts("\nrunning command #{bashCommand}")
+      `#{bashCommand}`
+    end
+  end
+end

data/lib/cnvrg/data.rb

@@ -128,6 +128,7 @@ module Cnvrg
       message = options["message"]
       cli.sync_data_new(new_branch, force, verbose, commit, all_files, tags, parallel, chunk_size, init, message)
     end
+
     desc 'data download', 'Download files from remote server'
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits", :default => false
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
@@ -151,27 +152,52 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
     method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
-
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     def clone(dataset_url)
-      #test
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
       commit =options[:commit]
       query =options[:query]
       read = options[:read]
       remote = options[:remote]
-      cli.clone_data(dataset_url, only_tree=only_tree,commit=commit, query=query, read=read, remote=remote, relative: options[:relative])
+      soft = options[:soft]
+      flatten = options[:flatten]
+      cli.clone_data(
+          dataset_url,
+          only_tree=only_tree,
+          commit=commit,
+          query=query,
+          read=read,
+          remote=remote,
+          flatten: flatten,
+          relative: options[:relative],
+          soft: soft
+      )
     end
 
     desc 'data verify DATASETS_TITLES', 'verify datasets', :hide => true
-    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => 15
-
+    method_option :timeout, :type => :numeric, :aliases => ["-t", "--timeout"], :desc => "Time to wait before returning final answer", :default => nil
     def verify(*dataset_titles)
       cli = Cnvrg::CLI.new()
       timeout =options[:timeout]
       cli.verify_datasets(dataset_titles, timeout)
     end
 
+    desc 'data scan', 'lookup datasets', :hide => true
+    def scan()
+      cli = Cnvrg::CLI.new()
+      cli.scan_datasets()
+    end
+
+    desc "data block DATASET_TITLES", 'verifying that datasets exists', hide: true
+    def block(*dataset_slugs)
+      not_verified = true
+      while not_verified
+        not_verified = dataset_slugs.select{|slug| not Dataset.verify_dataset(slug)}.present?
+      end
+    end
+
     desc 'data set --url=DATASET_URL', 'Set dataset url to other url'
     method_option :url, :type => :string, :aliases => ["--url"], :default => ''
     def set
@@ -186,22 +212,48 @@ module Cnvrg
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
     method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
+    method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
+    method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
       dir = options[:dir]
+      force = options["force"]
       commit = options[:commit]
       message = options[:message]
+      threads = options[:threads]
+      chunk_size = options[:chunk_size]
+      cli.data_put(
+          dataset_url,
+          files: files,
+          dir: dir,
+          commit: commit,
+          force: force,
+          threads: threads,
+          chunk_size: chunk_size,
+          message: message
+      )
+    end
 
-      cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
+    desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
+    method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    def rm(dataset_url, *regex_list)
+      cli = Cnvrg::CLI.new()
+      message = options[:message]
+      cli.data_rm(dataset_url, regex_list: regex_list, message: message)
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
     method_option :query, :type => :string, :aliases => ["-q", "--query"], :default => nil
+    method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
+    method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
     def clone_query(dataset_url)
       cli = Cnvrg::CLI.new()
-      query =options[:query]
-      cli.clone_data_query(dataset_url,query=query)
+      query = options[:query]
+      flatten = options[:flatten]
+      soft =options[:soft]
+      cli.clone_data_query(dataset_url,query=query, flatten, soft: soft)
     end
 
     desc 'data delete DATASET_SLUG', 'Delete dataset'
@@ -217,12 +269,13 @@ module Cnvrg
       cli.list_dataset()
 
     end
-    desc 'data commits', 'List all commits for a current dataset'
 
-    def commits()
+    desc 'data commits URL/SLUG', 'List all commits for a given dataset'
+    method_option :commit_sha1, :type => :string, :aliases => ["-c", "--commit"], :default => nil
+    def commits(dataset_url)
       cli = Cnvrg::CLI.new()
-      cli.list_dataset_commits()
-
+      commit_sha1 = options[:commit_sha1]
+      cli.list_dataset_commits(dataset_url, commit_sha1:commit_sha1)
     end
 
     desc 'data files DATASET_URL', 'Show list of dataset files'

data/lib/cnvrg/datafiles.rb

@@ -9,8 +9,8 @@ module Cnvrg
   class Datafiles
     ParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
-    LARGE_FILE = 1024 * 1024 * 5
-    MULTIPART_SPLIT = 10000000
+    LARGE_FILE=1024*1024*5
+    MULTIPART_SPLIT=10000000
 
     attr_reader :base_resource
 
@@ -25,7 +25,7 @@ module Cnvrg
 
     def refresh_storage_token
       current_time = Time.current
-      if current_time - @token_issue_time > 3.hours
+      if current_time - @token_issue_time > 1.hours
         @downloader = @dataset.get_storage_client
         @token_issue_time = Time.current
       end
@@ -33,7 +33,7 @@ module Cnvrg
 
     def check_file_sha1(filename, org_sha1, tag: 'conflict')
       file_loc = "#{Dir.pwd}/#{filename}"
-      sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
+      sha1 =  OpenSSL::Digest::SHA1.file(file_loc).hexdigest
       return 0 if sha1 == org_sha1
       FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
       return 1
@@ -52,7 +52,27 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.map { |p| p.gsub(/^\.\//, '') }
+      paths.map{|p| p.gsub(/^\.\//, '')}
+    end
+
+    def get_files_and_folders(paths)
+      files_and_folders = {}
+      paths.each do |file|
+        if File.exists? file
+          if File.directory? file
+            Dir.glob("#{file}/**/*").select do |f|
+              files_and_folders["#{f}/"] = "folder" if File.directory? f
+              files_and_folders[f] = "file" if File.file? f
+            end
+            files_and_folders["#{file}/"] = "folder"
+          else
+            files_and_folders[file] = "file"
+          end
+          next
+        end
+        raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
+      end
+      return files_and_folders
     end
 
     def check_files_sha1(files, resolver, tag)
@@ -83,11 +103,11 @@ module Cnvrg
       prefix = @dataset.get_dataset["bucket_prefix"]
       batch_size = 10000
       pbar = ProgressBar.create(:title => "Download Progress",
-                                :progress_mark => '=',
-                                :format => "%b%i| %c Files downloaded",
-                                :starting_at => 0,
-                                :total => nil,
-                                :autofinish => true)
+                                        :progress_mark => '=',
+                                        :format => "%b%i| %c Files downloaded",
+                                        :starting_at => 0,
+                                        :total => nil,
+                                        :autofinish => true)
       parallel_options = {
           in_threads: ParallelThreads,
           in_processes: Cnvrg::CLI::ParallelProcesses,
@@ -110,18 +130,36 @@ module Cnvrg
       end
     end
 
-
+    # This is for backwards compatibility only and should be removed in future versions:
     def put_commit(commit_sha1)
-      response = Cnvrg::API.request("#{@base_resource}/commit/latest", 'PUT', {commit_sha1: commit_sha1})
+      response = Cnvrg::API.request(
+          "#{@base_resource}/commit/latest",
+          'PUT',
+          {
+              commit_sha1: commit_sha1,
+              ignore: true # tells the new server to ignore this api call since its coming from the new CLI
+          }
+      )
       if response.present?
         msg = response['result']
       else
-        msg = "Cant save changes in the dataset"
+        msg =  "Can't save changes in the dataset"
       end
 
       Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
     end
 
+    def create_progressbar(title, total)
+      return ProgressBar.create(
+          :title => title,
+          :progress_mark => '=',
+          :format => "%b>>%i| %p%% %t",
+          :starting_at => 0,
+          :total => total,
+          :autofinish => true
+      )
+    end
+
 
     def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
       begin
@@ -158,6 +196,9 @@ module Cnvrg
         end
 
         files = results['files']
+
+        progressbar.progress += tree.keys.length - files.length if progressbar.present?
+        progress_semaphore = Mutex.new
         upload_error_files = []
         @temp_upload_progressbar.progress += tree.keys.length - files.length if @temp_upload_progressbar.present?
         Parallel.map((files.keys), in_threads: threads) do |k|
@@ -169,11 +210,12 @@ module Cnvrg
             tree.except!(k)
             Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
           end
-          @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
+          progress_semaphore.synchronize { progressbar.progress += 1 if progressbar.present? }
         end
-        blob_ids = files.values.map { |f| f['bv_id'] }
+
+        blob_ids = files.values.map {|f| f['bv_id']}
         if blob_ids.present?
-          dirs = tree.keys.select { |k| tree[k].nil? } || []
+          dirs = tree.keys.select {|k| tree[k].nil?} || []
           Cnvrg::Logger.info("Sending Upload files save")
           upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
           unless Cnvrg::CLI.is_response_success(upload_resp, false)
@@ -190,17 +232,231 @@ module Cnvrg
       end
     end
 
-    def upload_single_file(file)
+    def delete_multiple_files(commit_sha1, regex_list)
+      begin
+        Cnvrg::Logger.log_info("Sending Delete Files request")
+        resp = Cnvrg::API.request(
+            @base_resource + "delete_files",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                regex_list: regex_list,
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        Cnvrg::Logger.log_info("Delete Files request Successful")
+        return resp["files"], resp["folders"], resp["job_id"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
+      begin
+        resp = Cnvrg::API.request(
+            @base_resource + "delete_files_by_chunk",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                regex_list: regex_list,
+                chunk_size: chunk_size,
+                offset: offset
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        return resp["total_changes"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def get_delete_progress(commit_sha1, job_id)
+      begin
+        resp = Cnvrg::API.request(
+            @base_resource + "get_delete_progress",
+            'POST_JSON',
+            {
+                commit_sha1: commit_sha1,
+                job_id: job_id
+            }
+        )
+        unless Cnvrg::CLI.is_response_success(resp, false)
+          Cnvrg::Logger.log_method(bind: binding)
+          raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
+        end
+        return resp["total_deleted"]
+      rescue => e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+      retry_count = 0
+      loop do
+        upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {
+            commit_sha1: commit_sha1,
+            tree: tree,
+            force: force,
+            is_branch: new_branch,
+            partial_commit: partial_commit
+        })
+        if not (Cnvrg::CLI.is_response_success(upload_resp, false))
+          #Cnvrg::Logger.log_method(bind: binding)
+          retry_count += 1
+
+          puts "Failed request upload files: #{Time.current}"
+          puts upload_resp
+
+          if retry_count > 5
+            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          end
+          sleep 5
+          next
+        end
+        return upload_resp['result'].with_indifferent_access
+      end
+
+    end
+
+    def upload_multiple_files_optimized(files, commit_sha1, threads: ParallelThreads, chunk_size: 1000, force: false, new_branch: false, prefix: '', partial_commit: nil)
+      cli = CLI.new
+      cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
+
+      progressbar = create_progressbar("Upload Progress", files.size)
+      cli = CLI.new
+
+      # Vars to handle the parallelism
+      progress_mutex = Mutex.new
+      file_queue = Queue.new
+      progress_queue = Queue.new
+      worker_threads = []
+
+      # Vars to keep track of uploaded files and directories
+      dirs = []
+      uploaded_files = []
+
+      begin
+        # Init working threads that handle the upload of the files:
+        threads.times do |i|
+          worker_threads[i] = Thread.new do
+            # wait for file_queue.close to break the loop
+            while file = file_queue.deq
+              success = upload_single_file(cli, file)
+              file[:success] = success
+              if not success
+                cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
+                Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
+              end
+              progress_queue << file
+            end
+          end
+        end
+
+        # init the thread that handles the file upload progress and saving them in the server
+        progress_thread = Thread.new do
+          loop do
+            file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
+            unless file.nil?
+              progress_mutex.synchronize {
+                progressbar.progress += 1
+                uploaded_files.append(file) if file[:success]
+              }
+
+              if uploaded_files.size == chunk_size or progressbar.finished?
+                refresh_storage_token
+                # puts "progress: #{progress_queue.length}"
+                # puts "files: #{file_queue.length}"
+                Cnvrg::Logger.info("Sending Upload files save")
+                blob_ids = uploaded_files.map {|f| f['bv_id']}
+                upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
+                unless Cnvrg::CLI.is_response_success(upload_resp, false)
+                  Cnvrg::Logger.log_method(bind: binding)
+                  raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                end
+                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
+                uploaded_files = []
+                dirs = []
+              end
+            else
+              sleep(0.1)
+            end
+
+            if progressbar.finished?
+              # puts "finished"
+              file_queue.close()
+              progress_queue.close()
+              Thread.exit
+            end
+          end
+        end
+
+        file_chunks = files.each_slice(chunk_size).to_a
+        # Fetch the required files from the server:
+        Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
+
+          tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix)
+          results = request_upload_files(commit_sha1, tree, force, new_branch, partial_commit)
+
+          # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
+
+          if results['files'].blank?
+            progress_mutex.synchronize { progressbar.progress += tree.keys.length }
+            next
+          end
+
+          # Handle directories:
+          new_dirs = tree.keys.select {|k| tree[k].nil?}
+          dirs += new_dirs
+
+          files_to_upload = results['files']
+          progress_mutex.synchronize {
+            progressbar.progress += tree.keys.length - files_to_upload.length
+          }
+
+          files_to_upload.keys.each do |key|
+            while file_queue.size > 5000
+              sleep(0.1)
+            end
+            file_queue.push tree[key].merge(files_to_upload[key])
+          end
+        end
+
+        progress_thread.join()
+        worker_threads.each(&:join)
+
+      rescue => e
+        puts e
+        Cnvrg::Logger.log_method(bind: binding)
+        Cnvrg::Logger.log_error(e)
+        raise e
+      end
+    end
+
+    def upload_single_file(cli, file)
+      success = false
       begin
         file = file.as_json
         Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
         @downloader.safe_upload(file["path"], file["absolute_path"])
+        success = true
         Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
       rescue => e
         Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
         Cnvrg::Logger.log_error(e)
-        return false
       end
+      success
     end
 
     def upload_file(absolute_path, relative_path, commit_sha1)
@@ -208,12 +464,12 @@ module Cnvrg
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-      sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+      sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
       if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
         is_valid = false
         begin
-          content = open(absolute_path).read()
-          hash = YAML.load(open(absolute_path).read())
+        content = open(absolute_path).read()
+        hash = YAML.load(open(absolute_path).read())
           # if level 1 keys count is 1
           if hash.keys.count == 1
             if hash["tags"].present?
@@ -221,7 +477,7 @@ module Cnvrg
             elsif hash[hash.keys.first].class != Hash
               is_valid = true
             end
-            # if level 1 keys count is greater than 1
+          # if level 1 keys count is greater than 1
           elsif hash.keys.count > 1
             if hash["tags"].present? and hash["tags"].class == Hash
               is_valid = false
@@ -255,7 +511,6 @@ module Cnvrg
         return s3_res
       end
     end
-
     def upload_tar_file(absolute_path, relative_path, commit_sha1)
       begin
         file_name = File.basename relative_path
@@ -263,24 +518,24 @@ module Cnvrg
         mime_type = MimeMagic.by_path(absolute_path)
         content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
         begin
-          chunked_bytes = [100, (file_size * 0.01)].min
+          chunked_bytes = [100, (file_size*0.01)].min
           total_yanked = ""
           open(absolute_path, "rb") do |f|
             total_yanked = f.read(chunked_bytes)
           end
           if !total_yanked.empty?
-            sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
+            sha1 =  OpenSSL::Digest::SHA1.hexdigest(total_yanked)
           else
-            sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+            sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
           end
         rescue
-          sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
+          sha1 =  OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
         end
 
         upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                            commit_sha1: commit_sha1, file_name: file_name,
                                                                                            file_size: file_size, file_content_type: content_type, sha1: sha1,
-                                                                                           new_version: true})
+                                                                                            new_version:true})
         if Cnvrg::CLI.is_response_success(upload_resp, false)
           path = upload_resp["result"]["path"]
           s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -316,15 +571,14 @@ module Cnvrg
       return false
 
     end
-
-    def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
+    def upload_data_log_file(absolute_path, relative_path,data_commit_sha)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       content_type = "text/x-log"
       upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
-                                                                                              file_name: file_name, log_date: Time.now,
-                                                                                              file_size: file_size, file_content_type: content_type,
-                                                                                              data_commit_sha1: data_commit_sha})
+                                                                                             file_name: file_name, log_date: Time.now,
+                                                                                             file_size: file_size, file_content_type: content_type,
+                                                                                              data_commit_sha1:data_commit_sha})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
@@ -415,15 +669,15 @@ module Cnvrg
 
     end
 
-    def upload_cnvrg_image(absolute_path, image_name, secret)
+    def upload_cnvrg_image(absolute_path, image_name,secret)
       file_name = File.basename relative_path
       file_size = File.size(absolute_path).to_f
       mime_type = MimeMagic.by_path(absolute_path)
       content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
       upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
-                                                                                         file_name: file_name,
-                                                                                         file_size: file_size, file_content_type: content_type,
-                                                                                         secret: secret})
+                                                                                     file_name: file_name,
+                                                                                     file_size: file_size, file_content_type: content_type,
+                                                                                     secret: secret})
       if Cnvrg::CLI.is_response_success(upload_resp, false)
         path = upload_resp["result"]["path"]
         s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -467,7 +721,7 @@ module Cnvrg
         sts_path = upload_resp["result"]["path_sts"]
 
         retries = 0
-        success = false
+        success= false
         while !success and retries < 20
           begin
             if !Helpers.is_verify_ssl
@@ -477,7 +731,7 @@ module Cnvrg
             end
             success = true
           rescue => e
-            retries += 1
+            retries +=1
             sleep(5)
 
           end
@@ -489,33 +743,33 @@ module Cnvrg
         key = split[0]
         iv = split[1]
         is_s3 = upload_resp["result"]["is_s3"]
-        access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
+        access =  Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
 
-        secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
+        secret =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
 
-        session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
-        region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
+        session =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
+        region =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
 
-        bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
-        server_side_encryption = upload_resp["result"]["server_side_encryption"]
+        bucket =  Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
+        server_side_encryption =upload_resp["result"]["server_side_encryption"]
         use_accelerate_endpoint = false
         if is_s3 or is_s3.nil?
 
-          use_accelerate_endpoint = true
+          use_accelerate_endpoint =true
           client = Aws::S3::Client.new(
-              :access_key_id => access,
+              :access_key_id =>access,
               :secret_access_key => secret,
               :session_token => session,
               :region => region,
               :http_open_timeout => 60, :retry_limit => 20)
         else
-          endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
+          endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
           use_accelerate_endpoint = false
           client = Aws::S3::Client.new(
-              :access_key_id => access,
+              :access_key_id =>access,
               :secret_access_key => secret,
               :region => region,
-              :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+              :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
               :http_open_timeout => 60, :retry_limit => 20)
         end
 
@@ -527,11 +781,13 @@ module Cnvrg
         end
 
 
+
+
         s3 = Aws::S3::Resource.new(client: client)
 
         resp = s3.bucket(bucket).
-            object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
-            upload_file(file_path, options)
+            object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
+            upload_file(file_path,options)
 
 
         return resp
@@ -588,7 +844,6 @@ module Cnvrg
       response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
       return Cnvrg::CLI.is_response_success(response, false)
     end
-
     def download_list_files_in_query(response, dataset_home)
       sts_path = response["path_sts"]
       if !Helpers.is_verify_ssl
@@ -600,29 +855,29 @@ module Cnvrg
       key = split[0]
       iv = split[1]
 
-      access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
+      access =  Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
 
-      secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
+      secret =  Cnvrg::Helpers.decrypt(key,iv, response["sts_s"])
 
-      session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
-      region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
+      session =  Cnvrg::Helpers.decrypt(key,iv, response["sts_st"])
+      region =  Cnvrg::Helpers.decrypt(key,iv, response["region"])
 
-      bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
+      bucket =  Cnvrg::Helpers.decrypt(key,iv, response["bucket"])
       is_s3 = response["is_s3"]
       if is_s3 or is_s3.nil?
         client = Aws::S3::Client.new(
-            :access_key_id => access,
+            :access_key_id =>access,
             :secret_access_key => secret,
             :session_token => session,
             :region => region,
             :http_open_timeout => 60, :retry_limit => 20)
       else
-        endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
+        endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
         client = Aws::S3::Client.new(
-            :access_key_id => access,
+            :access_key_id =>access,
             :secret_access_key => secret,
             :region => region,
-            :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+            :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
             :http_open_timeout => 60, :retry_limit => 20)
       end
       list_files = response["files"]
@@ -639,18 +894,18 @@ module Cnvrg
       }
       download_count = 0
       Parallel.map((list_files), parallel_options) do |f|
-        file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
+        file_key =  Cnvrg::Helpers.decrypt(key,iv, f["path"])
         begin
-          begin
-            dir = File.dirname f["fullpath"]
-            FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
-          end
+        begin
+          dir = File.dirname f["fullpath"]
+          FileUtils.mkdir_p(dataset_home+"/"+ dir) unless File.exist? (dataset_home+"/"+ dir)
+        end
 
-          File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
-            resp = client.get_object({bucket: bucket,
-                                      key: file_key}, target: file)
-          end
-          download_count += 1
+        File.open(dataset_home+"/"+f["fullpath"], 'w+') do |file|
+          resp = client.get_object({bucket:bucket,
+                                    key:file_key}, target: file)
+        end
+        download_count += 1
         rescue
         end
 
@@ -662,95 +917,97 @@ module Cnvrg
       end
 
 
+
+
     end
 
 
-    def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
+    def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
+        res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 ,new_version:true, as_link:as_link})
         Cnvrg::CLI.is_response_success(res, false)
-        if res["result"]
-          file_url = res["result"]["file_url"]
+          if res["result"]
+            file_url = res["result"]["file_url"]
 
-          if as_link
-            return res["result"]
-          end
-          # begin
-          #   if !Helpers.is_verify_ssl
-          #     tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
-          #
-          #   else
-          #     tempfile = Down.download(file_url,open_timeout: 60)
-          #
-          #   end
-          #
-          #   FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
-          #   return true
-          # rescue
-          #
-          # end
-          download_resp = res
-          filename = download_resp["result"]["filename"]
+            if as_link
+              return res["result"]
+            end
+            # begin
+            #   if !Helpers.is_verify_ssl
+            #     tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
+            #
+            #   else
+            #     tempfile = Down.download(file_url,open_timeout: 60)
+            #
+            #   end
+            #
+            #   FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
+            #   return true
+            # rescue
+            #
+            # end
+            download_resp = res
+            filename = download_resp["result"]["filename"]
+
+            absolute_path += ".conflict" if conflict
+            sts_path = download_resp["result"]["path_sts"]
+            retries = 0
+            success= false
+            while !success and retries < 20
+              begin
+                if !Helpers.is_verify_ssl
+                  body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
+                else
+                  body = open(sts_path).read
+                end
+                success = true
+              rescue => e
+                retries +=1
+                sleep(5)
 
-          absolute_path += ".conflict" if conflict
-          sts_path = download_resp["result"]["path_sts"]
-          retries = 0
-          success = false
-          while !success and retries < 20
-            begin
-              if !Helpers.is_verify_ssl
-                body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
-              else
-                body = open(sts_path).read
               end
-              success = true
-            rescue => e
-              retries += 1
-              sleep(5)
-
             end
-          end
-          if !success
-            return false
-          end
-          split = body.split("\n")
-          key = split[0]
-          iv = split[1]
+            if !success
+              return false
+            end
+            split = body.split("\n")
+            key = split[0]
+            iv = split[1]
 
-          access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+            access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
 
-          secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
+            secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
 
-          session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
-          region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
+            session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+            region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
 
-          bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
-          file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
+            bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+            file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
 
-          is_s3 = download_resp["result"]["is_s3"]
-          if is_s3 or is_s3.nil?
-            client = Aws::S3::Client.new(
-                :access_key_id => access,
-                :secret_access_key => secret,
-                :session_token => session,
-                :region => region,
-                :http_open_timeout => 60, :retry_limit => 20)
-          else
-            endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
-            client = Aws::S3::Client.new(
-                :access_key_id => access,
-                :secret_access_key => secret,
-                :region => region,
-                :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
-                :http_open_timeout => 60, :retry_limit => 20)
-          end
+            is_s3 = download_resp["result"]["is_s3"]
+            if is_s3 or is_s3.nil?
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :session_token => session,
+                  :region => region,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            else
+              endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
+              client = Aws::S3::Client.new(
+                  :access_key_id =>access,
+                  :secret_access_key => secret,
+                  :region => region,
+                  :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
+                  :http_open_timeout => 60, :retry_limit => 20)
+            end
 
-          File.open(project_home + "/" + absolute_path, 'w+') do |file|
-            resp = client.get_object({bucket: bucket,
-                                      key: file_key}, target: file)
+            File.open(project_home+"/"+absolute_path, 'w+') do |file|
+              resp = client.get_object({bucket:bucket,
+                                        key:file_key}, target: file)
+            end
+            return true
           end
-          return true
-        end
 
       rescue => e
         return false
@@ -760,7 +1017,7 @@ module Cnvrg
 
     def download_data_file(commit_sha1, dataset_home)
       begin
-        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
+        res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
         Cnvrg::CLI.is_response_success(res, false)
         if res["result"]
           download_resp = res
@@ -777,38 +1034,38 @@ module Cnvrg
           iv = split[1]
 
 
-          access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
+          access =  Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
 
-          secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
+          secret =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
 
-          session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
-          region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
+          session =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
+          region =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
 
-          bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
-          file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
+          bucket =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
+          file_key =  Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
 
           is_s3 = download_resp["result"]["is_s3"]
           if is_s3 or is_s3.nil?
             client = Aws::S3::Client.new(
-                :access_key_id => access,
+                :access_key_id =>access,
                 :secret_access_key => secret,
                 :session_token => session,
                 :region => region,
                 :http_open_timeout => 60, :retry_limit => 20)
           else
-            endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
+            endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
             client = Aws::S3::Client.new(
-                :access_key_id => access,
+                :access_key_id =>access,
                 :secret_access_key => secret,
                 :region => region,
-                :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
+                :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
                 :http_open_timeout => 60, :retry_limit => 20)
           end
 
 
-          File.open(dataset_home + "/" + filename, 'w+') do |file|
+          File.open(dataset_home+"/"+filename, 'w+') do |file|
             resp = client.get_object({bucket: bucket,
-                                      key: file_key}, target: file)
+                                  key: file_key }, target: file)
           end
           return filename
         end
@@ -819,7 +1076,7 @@ module Cnvrg
       end
     end
 
-    def download_file(absolute_path, relative_path, project_home, conflict = false)
+    def download_file(absolute_path, relative_path, project_home, conflict=false)
       res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
       Cnvrg::CLI.is_response_success(res, false)
       if res["result"]
@@ -857,7 +1114,7 @@ module Cnvrg
       puts FileUtils.rmtree("#{absolute_path}")
     end
 
-    def revoke_download_file(absolute_path, filename, conflict = false)
+    def revoke_download_file(absolute_path, filename, conflict=false)
       begin
         file_location = absolute_path.gsub(/#{filename}\/?$/, "")
 
@@ -871,16 +1128,12 @@ module Cnvrg
 
     def revoke_download(tar_files, extracted_files)
       begin
-
         FileUtils.rm_rf(tar_files) unless (tar_files.nil? or tar_files.empty?)
         FileUtils.rm_rf(extracted_files) unless (extracted_files.nil? or extracted_files.empty?)
-
       rescue => e
         return false
       end
-
       return true
-
     end
 
     def delete_commit(commit_sha1)
@@ -895,17 +1148,29 @@ module Cnvrg
       return response
     end
 
-    def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
+    def start_commit(new_branch, force=false, chunks: 0, dataset: @dataset, message:nil)
       begin
         #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
         # create a new commit.
         idx = (force || new_branch) ? {} : dataset.get_idx
         commit = idx[:commit]
         next_commit = idx[:next_commit]
-        response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
-                                                                                username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
-        Cnvrg::CLI.is_response_success(response, true)
-        return response
+        response = Cnvrg::API.request(
+            "#{base_resource}/commit/start",
+            'POST',
+            {
+                dataset_slug: @dataset_slug,
+                new_branch: new_branch,
+                force:force,
+                username: @owner,
+                current_commit: commit,
+                next_commit: next_commit,
+                total_chunks: chunks,
+                message: message
+            }
+        )
+      Cnvrg::CLI.is_response_success(response, true)
+      return response
       rescue => e
         return false
       end
@@ -923,15 +1188,24 @@ module Cnvrg
       end
     end
 
-    def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
+    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
       begin
-        response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
+        response = Cnvrg::API.request(
+            "#{base_resource}/commit/end",
+            'POST',
+            {
+                commit_sha1: commit_sha1,
+                force:force,
+                success: success,
+                uploaded_files: uploaded_files,
+                commit_type: commit_type
+            }
+        )
         Cnvrg::CLI.is_response_success(response, true)
         return response
       rescue => e
         return false
       end
-
     end
 
     def end_commit_tar(commit_sha1, cur_idx)
@@ -951,13 +1225,13 @@ module Cnvrg
     end
 
     def get_trees(commit: "latest")
-      response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
+      response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
       return nil unless Cnvrg::CLI.is_response_success(response, false)
       response['result']['files']
     end
 
     def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
-      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
+      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
       unless Cnvrg::CLI.is_response_success(response, false)
         Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
         return nil
@@ -988,37 +1262,43 @@ module Cnvrg
       end
     end
 
-    def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
+    def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
       begin
-        last_chunk_size = chunk_size
-        q = {commit: commit, chunk_size: chunk_size}
-        overall = 0
-        while last_chunk_size > 0
-          response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
-          if Cnvrg::CLI.is_response_success(response, false)
-            files = response['files']
-            data_home = "#{Dir.pwd}/#{response['name']}"
-            last_chunk_size = files['keys'].length
-            break if last_chunk_size == 0
-            res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
-            overall += last_chunk_size
-            q[:latest] = files['latest']
-          else
-            last_chunk_size = 0
-          end
+      last_chunk_size = chunk_size
+      q = { commit: commit, chunk_size: chunk_size}
+      overall = 0
+      while last_chunk_size > 0
+        response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
+        if Cnvrg::CLI.is_response_success(response, false) 
+          files = response['files']
+          data_home = "#{Dir.pwd}/#{response['name']}"
+          last_chunk_size = files['keys'].length
+          break if last_chunk_size == 0
+          res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
+          overall += last_chunk_size
+          q[:latest] = files['latest']
+        else
+          last_chunk_size = 0
         end
-        Cnvrg::Result.new(true, "Cloned #{overall} files!")
+      end
+      Cnvrg::Result.new(true, "Cloned #{overall} files!")
       rescue => e
         Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
 
       end
     end
 
-    def generate_parallel_idx
-
+    def last_valid_commit()
+      begin
+        response = Cnvrg::API.request("#{base_resource}/last_valid_commit", 'GET')
+        Cnvrg::CLI.is_response_success(response, true)
+        return response
+      rescue => e
+        return false
+      end
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false)
       begin
         refresh_storage_token
         parallel_options = {
@@ -1027,14 +1307,16 @@ module Cnvrg
         }
         Parallel.map(files["keys"], parallel_options) do |f|
           begin
-            local_path = @dataset.local_path + '/' + f['name']
+            file_path = f['name']
+            file_path = File.basename(f['name']) if flatten
+            local_path = @dataset.local_path + '/' + file_path
             Cnvrg::Logger.log_info("Downloading #{local_path}")
             progressbar.progress += 1 if progressbar.present?
             if local_path.end_with? "/"
               @downloader.mkdir(local_path, recursive: true)
               next
             end
-            # blob
+              # blob
             local_path = "#{local_path}.conflict" if conflict
             storage_path = f["path"]
             if File.exists? local_path
@@ -1047,11 +1329,11 @@ module Cnvrg
             Cnvrg::Logger.log_error(e)
           end
         end
-        return Cnvrg::Result.new(true, "Downloaded successfully")
+        return Cnvrg::Result.new(true,"Downloaded successfully")
       rescue => e
-        Cnvrg::Logger.log_error(e)
-        return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
+          Cnvrg::Logger.log_error(e)
+          return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
+        end
       end
-    end
   end
 end