cnvrg 1.11.21 → 1.11.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc1bc67b5f62516c87e90a9b71d7d4921236eb8c13446b0c8ddac44ef9cf8544
-  data.tar.gz: '09b88539fc1fb041151d7fb1ec9931d0ddccc6e7cc50f958467d6e9fb976ba5d'
+  metadata.gz: a0ea5180b8920a78032ea60f9ccedea54357c3207720ff802b5c323aac6e4773
+  data.tar.gz: 54f0e631a050d0232921c0103aba65cd6c0ca48085161ee2137bf3ef975550ea
 SHA512:
-  metadata.gz: b7206c5c43fec47c6519e3fa9c798cbef0232ae0d9d70e708499d2312bbd5af802d6fc62e3b02f0f137f6322cd3486848fd9effbcbcca521171062ad22830e27
-  data.tar.gz: 27905ef2f218a241db92e78533390dc0b3b3fa6aa8242a925efc106eacf43d2adc11fdfc12f978ce1155fa6e1948ab9f98026ca163cb57db3729ee6080b15481
+  metadata.gz: 2805218f0460fe9bc1315f489d48fb37ea9795836f379eb5b6ab071c42a10ee76d16decb24c62c518356b35ca58489a9ba7637240c842459ad1ed6a94799633f
+  data.tar.gz: ed5bf8a1b5867c2ab6705dede44534074771387c31e8049c12f1cbf5f538c72e6cedbc910378db09ae6f2e1ca23524a54fd4109847c1aa02f4538a341ec4c68b

data/lib/cnvrg/api.rb

@@ -31,6 +31,10 @@ module Cnvrg
         end
         def self.request(resource, method = 'GET', data = {}, parse_request = true)
             resource = URI::encode resource
+
+            # We need to remoe all double slashes from the url to work with the proxy
+            resource = resource.gsub(/[\/]{2,}/, "/").gsub("https:/", "https://").gsub("http:/", "http://")
+
             begin
                 n = Netrc.read
             rescue => e

data/lib/cnvrg/cli.rb

@@ -858,7 +858,7 @@ module Cnvrg
     method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
     method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
-    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
+    def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15, cache_link: false)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
@@ -904,7 +904,7 @@ module Cnvrg
 
           commit = response["result"]["commit"]
           files_count = response["result"]["file_count"]
-          files = @files.get_clone_chunk(commit: commit)
+          files = @files.get_clone_chunk(commit: commit, cache_link: cache_link)
           downloaded_files = 0
           progressbar = ProgressBar.create(:title => "Download Progress",
                                            :progress_mark => '=',
@@ -917,7 +917,7 @@ module Cnvrg
 
           while files['keys'].length > 0
             Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
-            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
+            @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads, cache_link: cache_link)
 
             downloaded_files += files['keys'].length
             files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -1201,15 +1201,18 @@ module Cnvrg
     end
 
     desc '', '', :hide => true
-    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
+    def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil, auto_cache: false, external_disk: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
-
+        if auto_cache && external_disk.blank?
+          raise SignalException.new(1, "for auto caching external disk is required")
+        end
         owner, slug = get_owner_slug(dataset_url)
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
         @files = @datafiles.verify_files_exists(files)
+        @files = @files.uniq { |t| t.gsub('./', '')}
 
         if @files.blank?
           raise SignalException.new(1, "Cant find files to upload, exiting.")
@@ -1227,7 +1230,7 @@ module Cnvrg
           Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
+            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commit")
           end
           @commit = response['result']['sha1']
         else
@@ -1253,7 +1256,7 @@ module Cnvrg
           raise SignalException.new(1, res.msg)
         end
         Cnvrg::Logger.info("Saving commit on server")
-        res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
+        res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put", auto_cache: auto_cache, external_disk: external_disk)
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
         unless response.is_success?
@@ -1261,19 +1264,25 @@ module Cnvrg
         end
 
         log_message("Uploading files finished Successfully", Thor::Shell::Color::GREEN)
+        if msg['cache_error'].present?
+          log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
+        end
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false
       end
     end
 
-
     desc '', '', :hide => true
-    def data_rm(dataset_url, regex_list: [], commit: '', message: nil)
+    def data_rm(dataset_url, regex_list: [], commit: '', message: nil, auto_cache: false, external_disk: nil)
       begin
         verify_logged_in(false)
         log_start(__method__, args, options)
 
+        if auto_cache && external_disk.blank?
+          raise SignalException.new(1, "for auto caching external disk is required")
+        end
+
         owner, slug = get_owner_slug(dataset_url)
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
@@ -1309,7 +1318,7 @@ module Cnvrg
           offset += chunk_size
         end
 
-        res = @datafiles.end_commit(@commit,false, success: true)
+        res = @datafiles.end_commit(@commit,false, success: true, auto_cache: auto_cache, external_disk: external_disk)
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
         unless response.is_success?
@@ -1317,6 +1326,9 @@ module Cnvrg
         end
 
         log_message("Deleting files finished Successfully", Thor::Shell::Color::GREEN)
+        if msg['cache_error'].present?
+          log_message("Couldn't cache commit: #{msg['cache_error']}", Thor::Shell::Color::YELLOW)
+        end
       rescue SignalException => e
         log_message(e.message, Thor::Shell::Color::RED)
         return false
@@ -2308,7 +2320,6 @@ module Cnvrg
         @project = Project.new(get_project_home)
         chunk_size = chunk_size ? chunk_size : options["chunk_size"]
 
-
         # Enable local/experiment exception logging
         suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
         if in_exp
@@ -2346,7 +2357,6 @@ module Cnvrg
             log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
             return true
           end
-          force = true
         end
 
         if ignore.nil? or ignore.empty?

data/lib/cnvrg/data.rb

@@ -81,7 +81,6 @@ module Cnvrg
       end
     end
 
-
     desc "data upload", "Upload files from local dataset directory to remote server"
     method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
     method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
@@ -155,6 +154,7 @@ module Cnvrg
     method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
     method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
     method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
+    method_option :cache_link, :type => :boolean, :aliases => ["--cache_link"], :default => false, :hide => true
     def clone(dataset_url)
       cli = Cnvrg::CLI.new()
       only_tree =options[:only_tree]
@@ -165,6 +165,7 @@ module Cnvrg
       soft = options[:soft]
       flatten = options[:flatten]
       threads = options[:threads]
+      cache_link = options[:cache_link]
       cli.clone_data(
           dataset_url,
           only_tree=only_tree,
@@ -175,7 +176,8 @@ module Cnvrg
           flatten: flatten,
           relative: options[:relative],
           soft: soft,
-          threads: threads
+          threads: threads,
+          cache_link: cache_link
       )
     end
 
@@ -220,6 +222,8 @@ module Cnvrg
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
     method_option :chunk_size, :type => :numeric, :aliases => ["-cs","--chunk"], :default => 1000
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
+    method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
 
     def put(dataset_url, *files)
       cli = Cnvrg::CLI.new()
@@ -231,6 +235,8 @@ module Cnvrg
       message = options[:message]
       threads = options[:threads]
       chunk_size = options[:chunk_size]
+      auto_cache = options[:auto_cache]
+      external_disk = options[:external_disk]
       cli.data_put(
         dataset_url,
         files: files,
@@ -240,16 +246,28 @@ module Cnvrg
         override: override,
         threads: threads,
         chunk_size: chunk_size,
-        message: message
+        message: message,
+        auto_cache: auto_cache,
+        external_disk: external_disk
       )
     end
 
     desc 'data rm DATASET_URL FILES_PREFIX', 'Delete selected files from remote server'
     method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
+    method_option :auto_cache, :type => :boolean, :aliases => ["--auto_cache"], :desc => "auto_cache", :default => false
+    method_option :external_disk, :type => :string, :aliases => ["--external_disk"], :desc => "external_disk_title", :default => nil
     def rm(dataset_url, *regex_list)
       cli = Cnvrg::CLI.new()
       message = options[:message]
-      cli.data_rm(dataset_url, regex_list: regex_list, message: message)
+      auto_cache = options[:auto_cache]
+      external_disk = options[:external_disk]
+      cli.data_rm(
+        dataset_url,
+        regex_list: regex_list,
+        message: message,
+        auto_cache: auto_cache,
+        external_disk: external_disk
+      )
     end
 
     desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'

data/lib/cnvrg/datafiles.rb

@@ -47,6 +47,7 @@ module Cnvrg
         file = file[0..-2] if file.end_with? '/'
         if File.exists? file
           if File.directory? file
+            paths << file unless file == '.'
             paths += Dir.glob("#{file}/**/*")
           else
             paths << file
@@ -344,18 +345,20 @@ module Cnvrg
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
-      progressbar = create_progressbar("Upload Progress", files.size)
+      num_files = files.size
+      progressbar = create_progressbar("Upload Progress", num_files)
       cli = CLI.new
 
       # Vars to handle the parallelism
       progress_mutex = Mutex.new
       file_queue = Queue.new
       progress_queue = Queue.new
+      dirs_queue = Queue.new
       worker_threads = []
       progress_threads = []
+      old_api = false
 
       # Vars to keep track of uploaded files and directories
-      dirs = []
       uploaded_files = []
 
       begin
@@ -378,6 +381,36 @@ module Cnvrg
           end
         end
 
+        dir_thread = Thread.new do
+          dirs_to_create = []
+          loop do
+            dir = dirs_queue.deq(non_block: true) rescue nil
+            if dir.nil? && !progressbar.finished?
+              sleep 0.2
+              Cnvrg::Logger.info("directories thread status: progressbar.finished? #{progressbar.finished?} || dirs_queue.empty? #{dirs_queue.empty?} #{dirs_queue.size}  || dirs_to_create.empty? #{dirs_to_create.empty?} #{dirs_to_create.size}")
+            else
+              dirs_to_create << dir
+
+              if dirs_to_create.size >= 1000 || progressbar.finished?
+                resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", { dirs: dirs_to_create, commit_sha1: commit_sha1 })
+                Cnvrg::Logger.info("uploaded directories chunk, finished with #{resp}")
+                if resp == false # if resp is false it means 404 which is old server
+                  old_api = true
+                  break
+                end
+                unless Cnvrg::CLI.is_response_success(resp, false)
+                  dirs_to_create = []
+                  time = Time.current
+                  Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+                  next
+                end
+                dirs_to_create = []
+              end
+              break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
+            end
+          end
+        end
+
         # init the thread that handles the file upload progress and saving them in the server
         threads.times do |i|
           progress_threads[i] = Thread.new do
@@ -385,52 +418,46 @@ module Cnvrg
               file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
               unless file.nil?
                 blob_ids = []
-                dirs_to_upload = []
-
                 progress_mutex.synchronize {
                   progressbar.progress += 1
                   uploaded_files.append(file) if file[:success]
 
                   if uploaded_files.size >= chunk_size or progressbar.finished?
                     blob_ids = uploaded_files.map {|f| f['bv_id']}
-                    dirs_to_upload = dirs.clone
                     uploaded_files = []
-                    dirs = []
                   end
                 }
 
                 if blob_ids.present?
+                  random_id = (0...10).map { ('a'..'z').to_a[rand(26)] }.join
                   refresh_storage_token
-                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
-
-
+                  Cnvrg::Logger.info("chunk #{random_id}: Finished uploading chunk of #{chunk_size} files, Sending Upload files save")
                   retry_count = 0
                   loop do
-                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
 
                     if not (Cnvrg::CLI.is_response_success(upload_resp, false))
                       retry_count += 1
-                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
-                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      Cnvrg::Logger.log_error_message("chunk #{random_id}: Failed request save files: #{Time.current}, retry: #{retry_count}")
                       if retry_count > 20
-                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        puts "chunk #{random_id}: Failed to save files: #{Time.current}, trying next chunk"
                         break
                       end
                       sleep 5
                       next
                     end
-                    Cnvrg::Logger.info("Chunk saved on server")
+                    Cnvrg::Logger.info("chunk #{random_id}: Chunk saved on server")
                     break
                   end
                 end
               else
                 sleep(0.1)
               end
-
+              Cnvrg::Logger.info("progress_threads: progressbar.finished? #{progressbar.finished?}")
               if progressbar.finished?
                 Cnvrg::Logger.info("Progress bar finished closing queues")
-                file_queue.close()
-                progress_queue.close()
+                file_queue.close
+                progress_queue.close
                 Thread.exit
               end
             end
@@ -439,24 +466,43 @@ module Cnvrg
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
+        num_chunks = (num_files / 1000.0).ceil
+        chunk_index = 0
         Parallel.map((file_chunks), in_threads: threads) do |chunk|
-          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
-          Cnvrg::Logger.info("Generating chunk idx")
+          chunk_index += 1
+          self_chunk_index = chunk_index
+          files_chunk = chunk.map { |p| p.gsub(/^\.\//, '') }
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
-          Cnvrg::Logger.info("Getting files info from server")
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Generating chunk idx")
+
+          # Handle directories:
+          unless old_api
+            while dirs_queue.size > 5000
+              sleep(0.1)
+            end
+          end
+          new_dirs = tree.keys.select { |k| tree[k].nil? }
+          if new_dirs.blank?
+            ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
+            file = tree.keys.find { |k| tree[k] != nil }
+            dirs_queue.push(file) unless old_api
+          end
+          new_dirs.each { |dir| dirs_queue.push dir }
+
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Getting files info from server")
           next unless results
 
           if results['files'].blank?
+            Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: no files to upload skipping chunk")
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
           end
 
-          # Handle directories:
-          new_dirs = tree.keys.select {|k| tree[k].nil?}
-          dirs += new_dirs
-
           files_to_upload = results['files']
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: number of files to upload in this chunk: #{tree.keys.length - files_to_upload.length}")
           progress_mutex.synchronize {
             progressbar.progress += tree.keys.length - files_to_upload.length
           }
@@ -468,8 +514,13 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+
+        Cnvrg::Logger.info("Waiting dir_thread to finish")
+        dir_thread.join
+        dirs_queue.close
+        Cnvrg::Logger.info("Waiting progress_thread to finish")
         progress_threads.each(&:join)
+        Cnvrg::Logger.info("Waiting workers to finish")
         worker_threads.each(&:join)
         Thread.report_on_exception = true
       rescue => e
@@ -1209,7 +1260,7 @@ module Cnvrg
       false
     end
 
-    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil)
+    def end_commit(commit_sha1, force, success: true, uploaded_files: 0, commit_type: nil, auto_cache: false, external_disk: nil)
       counter = 0
       begin
         counter += 1
@@ -1221,7 +1272,9 @@ module Cnvrg
                 force:force,
                 success: success,
                 uploaded_files: uploaded_files,
-                commit_type: commit_type
+                commit_type: commit_type,
+                auto_cache: auto_cache,
+                external_disk: external_disk
             }
         )
         is_success = Cnvrg::CLI.is_response_success(response, false)
@@ -1255,8 +1308,8 @@ module Cnvrg
       response['result']['files']
     end
 
-    def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
-      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
+    def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest', cache_link: false)
+      response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id, cache_link: cache_link})
       unless Cnvrg::CLI.is_response_success(response, false)
         Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
         return nil
@@ -1323,7 +1376,7 @@ module Cnvrg
       end
     end
 
-    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
+    def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15, cache_link: false)
       begin
         refresh_storage_token
         parallel_options = {
@@ -1344,10 +1397,18 @@ module Cnvrg
               # blob
             local_path = "#{local_path}.conflict" if conflict
             storage_path = f["path"]
-            # if File.exists? local_path
-            #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
-            #   next
-            # end
+              # if File.exists? local_path
+              #   Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
+              #   next
+              # end
+            if cache_link
+              cached_commits = f['cached_commits']
+
+              if cached_commits.present?
+                next if @downloader.link_file(cached_commits, local_path, @dataset.title, f['name'])
+              end
+            end
+
             resp = @downloader.safe_download(storage_path, local_path)
             Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
           rescue => e

data/lib/cnvrg/dataset.rb

@@ -564,7 +564,8 @@ module Cnvrg
         safe_path = file
         safe_path = file[1..-1] if file.start_with? "/"
 
-        label = safe_path.gsub(self.local_path + "/", "")
+        dataset_local_path = self.local_path + "/"
+        label = safe_path.start_with?(dataset_local_path) ? safe_path.sub(dataset_local_path, "") : safe_path
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           if cli
@@ -598,6 +599,7 @@ module Cnvrg
           }
         end
       end
+
       if prefix.present? #add the prefix as dirs to the files
         #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list
         dirs = prefix.split('/')

data/lib/cnvrg/downloader/client.rb

@@ -37,6 +37,21 @@ module Cnvrg
         ### need to be implemented..
       end
 
+      def link_file(cached_commits, local_path, dataset_title, file_name)
+        prepare_download(local_path)
+        cached_commits.each do |cached_commit|
+          nfs_path = "/nfs-disk/#{cached_commit}/#{dataset_title}/#{file_name}"
+          if File.exist? nfs_path
+            FileUtils.ln(nfs_path, local_path)
+            return true
+          end
+        end
+        false
+      rescue => e
+        Cnvrg::Logger.log_error(e)
+        false
+      end
+
       def safe_download(storage_path, local_path, decrypt: true)
         safe_operation(local_path) { self.download(storage_path, local_path, decrypt: decrypt) }
       end

data/lib/cnvrg/files.rb

@@ -134,7 +134,6 @@ module Cnvrg
           end
         end
 
-
       blob_ids.concat blob_id_chunk
     end
 

data/lib/cnvrg/project.rb

@@ -448,8 +448,8 @@ module Cnvrg
           next
         end
         if File.directory? e
-
-          tree_idx[label + "/"] = nil
+          dir_name = (label.ends_with? "/") ? label : (label + "/")
+          tree_idx[dir_name] = nil
         else
           file_in_idx = old_idx[:tree][label] rescue nil
           last_modified = File.mtime(e).to_f
@@ -513,6 +513,7 @@ module Cnvrg
         #upload
         local_idx = self.generate_idx(deploy: deploy, files: specific_files)
       end
+
       commit = local_idx[:commit]
       tree = local_idx[:tree]
       ignore_list = self.send_ignore_list()
@@ -521,12 +522,12 @@ module Cnvrg
         if tree.present?
           added += local_idx[:tree].keys
         end
-        response = {"result" => {"commit" => nil, "tree" => {"added" => added,
-                                                             "updated_on_server" => [],
-                                                             "updated_on_local" => [],
-                                                             "update_local" => [],
-                                                             "deleted" => [],
-                                                             "conflicts" => []}}}
+        response = { "result" => { "commit" => nil, "tree" => { "added" => added,
+                                                                "updated_on_server" => [],
+                                                                "updated_on_local" => [],
+                                                                "update_local" => [],
+                                                                "deleted" => [],
+                                                                "conflicts" => [] } } }
         return response
       end
       #we dont want to send it on download - we only compare between commits sha1 in download.
@@ -534,6 +535,7 @@ module Cnvrg
         #the new server doesnt need the tree, but the old probably needs :X
         local_idx[:tree] = {} if Cnvrg::Helpers.server_version > 0
       end
+
       response = Cnvrg::API.request(@base_resource + "status", 'POST', {idx: local_idx, new_branch: new_branch,
                                                                         current_commit: commit, ignore: ignore_list, force: force, in_exp: in_exp, download: download})
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.21'
-end
+  VERSION = '1.11.27'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.21
+  version: 1.11.27
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-03-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.9
+rubygems_version: 3.1.2
 signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.