cnvrg 1.11.26 → 1.11.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5b1b849c4b05132d0dbdd9f23e767ecd0080b7ae01a96a9aad5bd5b0c7327e6
-  data.tar.gz: e4d57db1c3dede3fe7907ade6be0787d950d5b17db380c7032b41a7df12de577
+  metadata.gz: a0ea5180b8920a78032ea60f9ccedea54357c3207720ff802b5c323aac6e4773
+  data.tar.gz: 54f0e631a050d0232921c0103aba65cd6c0ca48085161ee2137bf3ef975550ea
 SHA512:
-  metadata.gz: 5a8755391132d2f30c2a7c35ab4fb3c12bbb3a6f3e29df282ca884c4a8917c03944e45639b5b03f9b1a576d65c7d57c9f3323d969271cff15ce973a0f5b42b66
-  data.tar.gz: 0e22fb319eee5086097ed6108c21e333048ac443b9a1b4813475eb1932aa0857aba97722cdef42a9c4fe0d26784c8a95efd628e5ac4def95f785458750851b95
+  metadata.gz: 2805218f0460fe9bc1315f489d48fb37ea9795836f379eb5b6ab071c42a10ee76d16decb24c62c518356b35ca58489a9ba7637240c842459ad1ed6a94799633f
+  data.tar.gz: ed5bf8a1b5867c2ab6705dede44534074771387c31e8049c12f1cbf5f538c72e6cedbc910378db09ae6f2e1ca23524a54fd4109847c1aa02f4538a341ec4c68b

data/lib/cnvrg/datafiles.rb

@@ -345,7 +345,8 @@ module Cnvrg
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
-      progressbar = create_progressbar("Upload Progress", files.size)
+      num_files = files.size
+      progressbar = create_progressbar("Upload Progress", num_files)
       cli = CLI.new
 
       # Vars to handle the parallelism
@@ -355,6 +356,7 @@ module Cnvrg
       dirs_queue = Queue.new
       worker_threads = []
       progress_threads = []
+      old_api = false
 
       # Vars to keep track of uploaded files and directories
       uploaded_files = []
@@ -382,23 +384,30 @@ module Cnvrg
         dir_thread = Thread.new do
           dirs_to_create = []
           loop do
-            progress_mutex.synchronize {
-              dir = dirs_queue.deq(non_block: true) rescue nil
-              dirs_to_create << dir unless dir.nil?
-            }
-            if dirs_to_create.size >= 1000 || progressbar.finished?
-              resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", {dirs: dirs_to_create, commit_sha1: commit_sha1})
-
-              break if resp == false # if resp is false it means 404 which is old server
-              unless Cnvrg::CLI.is_response_success(resp, false)
-                time = Time.current
-                Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+            dir = dirs_queue.deq(non_block: true) rescue nil
+            if dir.nil? && !progressbar.finished?
+              sleep 0.2
+              Cnvrg::Logger.info("directories thread status: progressbar.finished? #{progressbar.finished?} || dirs_queue.empty? #{dirs_queue.empty?} #{dirs_queue.size}  || dirs_to_create.empty? #{dirs_to_create.empty?} #{dirs_to_create.size}")
+            else
+              dirs_to_create << dir
+
+              if dirs_to_create.size >= 1000 || progressbar.finished?
+                resp = Cnvrg::API.request(@base_resource + "create_dirs", "POST", { dirs: dirs_to_create, commit_sha1: commit_sha1 })
+                Cnvrg::Logger.info("uploaded directories chunk, finished with #{resp}")
+                if resp == false # if resp is false it means 404 which is old server
+                  old_api = true
+                  break
+                end
+                unless Cnvrg::CLI.is_response_success(resp, false)
+                  dirs_to_create = []
+                  time = Time.current
+                  Cnvrg::Logger.log_error_message("Failed to create dirs: #{time}, #{resp.try(:fetch, "message")}")
+                  next
+                end
                 dirs_to_create = []
-                next
               end
-              dirs_to_create = []
+              break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
             end
-            break if progressbar.finished? && dirs_queue.empty? && dirs_to_create.empty?
           end
         end
 
@@ -409,7 +418,6 @@ module Cnvrg
               file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
               unless file.nil?
                 blob_ids = []
-
                 progress_mutex.synchronize {
                   progressbar.progress += 1
                   uploaded_files.append(file) if file[:success]
@@ -421,32 +429,31 @@ module Cnvrg
                 }
 
                 if blob_ids.present?
+                  random_id = (0...10).map { ('a'..'z').to_a[rand(26)] }.join
                   refresh_storage_token
-                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
-
+                  Cnvrg::Logger.info("chunk #{random_id}: Finished uploading chunk of #{chunk_size} files, Sending Upload files save")
                   retry_count = 0
                   loop do
                     upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids})
 
                     if not (Cnvrg::CLI.is_response_success(upload_resp, false))
                       retry_count += 1
-                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
-                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      Cnvrg::Logger.log_error_message("chunk #{random_id}: Failed request save files: #{Time.current}, retry: #{retry_count}")
                       if retry_count > 20
-                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        puts "chunk #{random_id}: Failed to save files: #{Time.current}, trying next chunk"
                         break
                       end
                       sleep 5
                       next
                     end
-                    Cnvrg::Logger.info("Chunk saved on server")
+                    Cnvrg::Logger.info("chunk #{random_id}: Chunk saved on server")
                     break
                   end
                 end
               else
                 sleep(0.1)
               end
-
+              Cnvrg::Logger.info("progress_threads: progressbar.finished? #{progressbar.finished?}")
               if progressbar.finished?
                 Cnvrg::Logger.info("Progress bar finished closing queues")
                 file_queue.close
@@ -459,35 +466,43 @@ module Cnvrg
 
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
+        num_chunks = (num_files / 1000.0).ceil
+        chunk_index = 0
         Parallel.map((file_chunks), in_threads: threads) do |chunk|
-          files_chunk = chunk.map{|p| p.gsub(/^\.\//, '')}
-          Cnvrg::Logger.info("Generating chunk idx")
+          chunk_index += 1
+          self_chunk_index = chunk_index
+          files_chunk = chunk.map { |p| p.gsub(/^\.\//, '') }
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads, cli: cli)
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Generating chunk idx")
 
-          progress_mutex.synchronize {
-            # Handle directories:
-            new_dirs = tree.keys.select { |k| tree[k].nil? }
-
-            if new_dirs.blank?
-              ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
-              file = tree.keys.find { |k| tree[k] != nil }
-              dirs_queue.push file
+          # Handle directories:
+          unless old_api
+            while dirs_queue.size > 5000
+              sleep(0.1)
             end
+          end
+          new_dirs = tree.keys.select { |k| tree[k].nil? }
+          if new_dirs.blank?
+            ## we need to send 1 file so we will inflated dirs from in case when we dont have folders in the tree
+            file = tree.keys.find { |k| tree[k] != nil }
+            dirs_queue.push(file) unless old_api
+          end
+          new_dirs.each { |dir| dirs_queue.push dir }
 
-            new_dirs.each { |dir| dirs_queue.push dir }
-          }
-          Cnvrg::Logger.info("Getting files info from server")
-
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: Finished Getting files info from server")
           next unless results
 
           if results['files'].blank?
+            Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: no files to upload skipping chunk")
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
           end
 
           files_to_upload = results['files']
-
+          Cnvrg::Logger.info("chunk #{self_chunk_index} / #{num_chunks}: number of files to upload in this chunk: #{tree.keys.length - files_to_upload.length}")
           progress_mutex.synchronize {
             progressbar.progress += tree.keys.length - files_to_upload.length
           }
@@ -500,10 +515,12 @@ module Cnvrg
           end
         end
 
-        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        Cnvrg::Logger.info("Waiting dir_thread to finish")
         dir_thread.join
         dirs_queue.close
+        Cnvrg::Logger.info("Waiting progress_thread to finish")
         progress_threads.each(&:join)
+        Cnvrg::Logger.info("Waiting workers to finish")
         worker_threads.each(&:join)
         Thread.report_on_exception = true
       rescue => e

data/lib/cnvrg/files.rb

@@ -134,7 +134,6 @@ module Cnvrg
           end
         end
 
-
       blob_ids.concat blob_id_chunk
     end
 

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.11.26'
+  VERSION = '1.11.27'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.11.26
+  version: 1.11.27
 platform: ruby
 authors:
 - Yochay Ettun