cnvrg 1.11.23 → 1.11.24

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2b8064702c190a7af0e1541f7a523e950e771439caa1f68d98177c094551dbb5
-  data.tar.gz: 982aa81d2da514f30b45ad45e4ec4b9a12a648e22c9b60d98afe318eba30e753
+  metadata.gz: e8a5d09f38ee4ce25caecdaf73b77e37fadff3b745454030cb15863210c1b319
+  data.tar.gz: 1f3e27bc4fa778ff048e7257f2ce753009ceaf0801c4ca3f4facfdc3011ad940
 SHA512:
-  metadata.gz: 9f3dcc03516cbe0a3ffd8bd5fa57f976b09028568e8014f161d43581fbc4b8c4def858297b55ef1d750efcd0af276905c006e5ebe7f3e17dd1268e81ad34f084
-  data.tar.gz: de7a4ea731e3230c225149d345e0ab2dde8466b2dd66008aba09d90a0bc287a9d61a8fb266f755a4665ce57c27e7c97c198b7435ab15c0d1fb63977552677150
+  metadata.gz: 14d79f14723eff1d8df81907183fabbe97270fd49610d4aa456fa820e4c19994c3d2bd1b7c64160f4d66a0636ff385a32b86f6fed2bedf9c0ed98c7968267c23
+  data.tar.gz: 80b4644fc6f560e54280ab726983f4e5d89978255991b29c510452b5514019025a3dcae45e4ad5f9a36e97823909eb6d1f2ee892cc63ef5090fa78ecacf595fe

data/lib/cnvrg/api.rb

@@ -31,6 +31,10 @@ module Cnvrg
         end
         def self.request(resource, method = 'GET', data = {}, parse_request = true)
             resource = URI::encode resource
+
+            # We need to remoe all double slashes from the url to work with the proxy
+            resource = resource.gsub(/[\/]{2,}/, "/").gsub("https:/", "https://").gsub("http:/", "http://")
+
             begin
                 n = Netrc.read
             rescue => e

data/lib/cnvrg/cli.rb

@@ -1210,6 +1210,7 @@ module Cnvrg
         @dataset = Dataset.new(dataset_info: {:owner =>  owner, :slug => slug})
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
         @files = @datafiles.verify_files_exists(files)
+        @files = @files.uniq { |t| t.gsub('./', '')}
 
         if @files.blank?
           raise SignalException.new(1, "Cant find files to upload, exiting.")

data/lib/cnvrg/datafiles.rb

@@ -47,6 +47,7 @@ module Cnvrg
         file = file[0..-2] if file.end_with? '/'
         if File.exists? file
           if File.directory? file
+            paths << file unless file == '.'
             paths += Dir.glob("#{file}/**/*")
           else
             paths << file
@@ -351,11 +352,11 @@ module Cnvrg
       progress_mutex = Mutex.new
       file_queue = Queue.new
       progress_queue = Queue.new
+      dirs_queue = Queue.new
       worker_threads = []
       progress_threads = []
 
       # Vars to keep track of uploaded files and directories
-      dirs = []
       uploaded_files = []
 
       begin
@@ -378,6 +379,29 @@ module Cnvrg
           end
         end
 
+        dir_thread = Thread.new do
+          dirs_to_create = []
+          loop do
+            progress_mutex.synchronize {
+              dir = dirs_queue.deq(non_block: true) rescue nil
+              dirs_to_create << dir unless dir.nil?
+            }
+            if dirs_to_create.size >= 1000 || progressbar.finished?
+              resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", {dirs: dirs_to_create, commit_sha1: commit_sha1})
+
+              break if resp == false # if resp is false it means 404 which is old server
+              unless Cnvrg::CLI.is_response_success(resp, false)
+                time = Time.current
+                Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+                dirs_to_create = []
+                next
+              end
+              dirs_to_create = []
+            end
+            break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
+          end
+        end
+
         # init the thread that handles the file upload progress and saving them in the server
         threads.times do |i|
           progress_threads[i] = Thread.new do
@@ -385,7 +409,6 @@ module Cnvrg
               file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
               unless file.nil?
                 blob_ids = []
-                dirs_to_upload = []
 
                 progress_mutex.synchronize {
                   progressbar.progress += 1
@@ -393,9 +416,7 @@ module Cnvrg
 
                   if uploaded_files.size >= chunk_size or progressbar.finished?
                     blob_ids = uploaded_files.map {|f| f['bv_id']}
-                    dirs_to_upload = dirs.clone
                     uploaded_files = []
-                    dirs = []
                   end
                 }
 
@@ -403,10 +424,9 @@ module Cnvrg
                   refresh_storage_token
                   Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
 
-
                   retry_count = 0
                   loop do
-                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
 
                     if not (Cnvrg::CLI.is_response_success(upload_resp, false))
                       retry_count += 1
@@ -429,8 +449,8 @@ module Cnvrg
 
               if progressbar.finished?
                 Cnvrg::Logger.info("Progress bar finished closing queues")
-                file_queue.close()
-                progress_queue.close()
+                file_queue.close
+                progress_queue.close
                 Thread.exit
               end
             end
@@ -443,7 +463,21 @@ module Cnvrg
           files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
           Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
+
+          progress_mutex.synchronize {
+            # Handle directories:
+            new_dirs = tree.keys.select { |k| tree[k].nil? }
+
+            if new_dirs.blank?
+              ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
+              file = tree.keys.find { |k| tree[k] != nil }
+              dirs_queue.push file
+            end
+
+            new_dirs.each { |dir| dirs_queue.push dir }
+          }
           Cnvrg::Logger.info("Getting files info from server")
+
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
           next unless results
 
@@ -452,11 +486,8 @@ module Cnvrg
             next
           end
 
-          # Handle directories:
-          new_dirs = tree.keys.select {|k| tree[k].nil?}
-          dirs += new_dirs
-
           files_to_upload = results['files']
+
           progress_mutex.synchronize {
             progressbar.progress += tree.keys.length - files_to_upload.length
           }
@@ -468,7 +499,10 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
+
         Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        dir_thread.join
+        dirs_queue.close
         progress_threads.each(&:join)
         worker_threads.each(&:join)
         Thread.report_on_exception = true

data/lib/cnvrg/dataset.rb

@@ -564,7 +564,8 @@ module Cnvrg
         safe_path = file
         safe_path = file[1..-1] if file.start_with? "/"
 
-        label = safe_path.gsub(self.local_path + "/", "")
+        dataset_local_path = self.local_path + "/"
+        label = safe_path.start_with?(dataset_local_path) ? safe_path.sub(dataset_local_path, "") : safe_path
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           if cli
@@ -598,6 +599,7 @@ module Cnvrg
           }
         end
       end
+
       if prefix.present? #add the prefix as dirs to the files
         #lets say the prefix is a/b/c so we want that a/, a/b/, a/b/c/ will be in our files_list
         dirs = prefix.split('/')

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.23'
-end
+  VERSION = '1.11.24'
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.23
+  version: 1.11.24
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-17 00:00:00.000000000 Z
+date: 2021-02-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler