cnvrg 1.10.12 → 1.10.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9c519291dee531a33698f1737d8019b9a9affff30c00c60ebb5904cf2ab4daa
-  data.tar.gz: 1dd54d29c1f0b7db384bfe62ae6f3678a118375515d6da451ef58094adfa334d
+  metadata.gz: 057f3283dffe548f882ed6cbdcfbc176e12fa75291f8a4867c8eee74fefd701f
+  data.tar.gz: fe30a40d86418cca0c98156ed89212ccfb159ead16dd248388a05f9d5bb3a05f
 SHA512:
-  metadata.gz: d4ce4dea833009867b132bdf6db3fa79bcc7396e828d2f7ad05433ca5c73bf06a999eb40d75d8895f8ae4108d7d6fb30fdc3149001edd63a25f785956abb2472
-  data.tar.gz: 1082b5f3dc5da602bb2089a97dec85774bed3a43c05c9247acf64c0bff7a6d50a641dee92f8a4b8999dd3732c5d3c75eec14c6e4b578c9b09d49f5bab7452e39
+  metadata.gz: 5b5ccc6f603a06affd160bc928c8ee99acf045eccf3905dcc1c23ceca8b4efdb185c3f78716bb111973df0420a5356dbf3565c52a9fe298f4fccdc886f3b07d7
+  data.tar.gz: 1cc6cbaa4737fb0151bed559352292908479ffce6e2bb1ba65812aef19d91b935265ec80f8a2a0f52111c02b3944aecb79d89e8d72518f4db094c75766834228

data/lib/cnvrg/cli.rb

@@ -1194,7 +1194,6 @@ module Cnvrg
     desc '', '', :hide => true
     def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, force: false, override: false, threads: 15, message: nil)
       begin
-        verify_logged_in(false)
         log_start(__method__, args, options)
 
         owner, slug = get_owner_slug(dataset_url)
@@ -1208,15 +1207,17 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
+          Cnvrg::Logger.info("Creating commit")
           response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
           @commit =  response['result']['commit_sha1']
         elsif commit.eql? "latest"
+          Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
-            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
+            raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commit")
           end
           @commit = response['result']['sha1']
         else
@@ -1233,15 +1234,15 @@ module Cnvrg
           override: override,
           chunk_size: chunk_size,
           prefix: dir,
-          threads: threads
+          threads: threads,
         )
-
+        Cnvrg::Logger.info("Finished upload files")
         # This is for backwards compatibility only and should be removed in future versions:
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
           raise SignalException.new(1, res.msg)
         end
-
+        Cnvrg::Logger.info("Saving commit on server")
         res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
@@ -3094,6 +3095,11 @@ module Cnvrg
       emails = options["emails"]
       wait = options["wait"]
 
+      if wait && local
+        log_message("WARN: `wait` option is not valid for local experiment, ignoring it", Thor::Shell::Color::YELLOW)
+        wait = false
+      end
+
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
         exit(1)
@@ -3635,7 +3641,8 @@ module Cnvrg
                   sleep 3
                 else
                   result.each do |slug, value|
-                    puts "Experiment #{slug} was exited with status #{value['exit_status']}"
+                    exit_status = value['exit_status']
+                    puts "Experiment #{slug} was exited with status #{exit_status}"
                   end
                   break
                 end
@@ -5301,8 +5308,8 @@ module Cnvrg
               count += 1
             end
             if File.exist? logfile_old
-              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              #@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              #@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
               FileUtils.remove logfile_old
             end
 

data/lib/cnvrg/data.rb

@@ -232,15 +232,15 @@ module Cnvrg
       threads = options[:threads]
       chunk_size = options[:chunk_size]
       cli.data_put(
-          dataset_url,
-          files: files,
-          dir: dir,
-          commit: commit,
-          force: force,
-          override: override,
-          threads: threads,
-          chunk_size: chunk_size,
-          message: message
+        dataset_url,
+        files: files,
+        dir: dir,
+        commit: commit,
+        force: force,
+        override: override,
+        threads: threads,
+        chunk_size: chunk_size,
+        message: message
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -52,7 +52,10 @@ module Cnvrg
         end
         raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
       end
-      paths.map{|p| p.gsub(/^\.\//, '')}
+      paths.each_slice(1000) do |chunk|
+        chunk.each { |p| p.gsub!(/^\.\//, '') }
+      end
+      paths
     end
 
     def get_files_and_folders(paths)
@@ -316,21 +319,21 @@ module Cnvrg
           #Cnvrg::Logger.log_method(bind: binding)
           retry_count += 1
 
-          puts "Failed request upload files: #{Time.current}"
-          puts upload_resp
+          Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
 
-          if retry_count > 5
-            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          if retry_count > 20
+            puts "Failed to upload files: #{Time.current}, trying next chunk"
+            return nil
           end
           sleep 5
           next
         end
         return upload_resp['result'].with_indifferent_access
       end
-
     end
 
     def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
+      Thread.report_on_exception = false
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
@@ -342,6 +345,7 @@ module Cnvrg
       file_queue = Queue.new
       progress_queue = Queue.new
       worker_threads = []
+      progress_threads = []
 
       # Vars to keep track of uploaded files and directories
       dirs = []
@@ -359,45 +363,69 @@ module Cnvrg
                 cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
                 Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
               end
+              while progress_queue.size > 15000
+                sleep(0.1)
+              end
               progress_queue << file
             end
           end
         end
 
         # init the thread that handles the file upload progress and saving them in the server
-        progress_thread = Thread.new do
-          loop do
-            file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
-            unless file.nil?
-              progress_mutex.synchronize {
-                progressbar.progress += 1
-                uploaded_files.append(file) if file[:success]
-              }
-
-              if uploaded_files.size == chunk_size or progressbar.finished?
-                refresh_storage_token
-                # puts "progress: #{progress_queue.length}"
-                # puts "files: #{file_queue.length}"
-                Cnvrg::Logger.info("Sending Upload files save")
-                blob_ids = uploaded_files.map {|f| f['bv_id']}
-                upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-                unless Cnvrg::CLI.is_response_success(upload_resp, false)
-                  Cnvrg::Logger.log_method(bind: binding)
-                  raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        threads.times do |i|
+          progress_threads[i] = Thread.new do
+            loop do
+              file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
+              unless file.nil?
+                blob_ids = []
+                dirs_to_upload = []
+
+                progress_mutex.synchronize {
+                  progressbar.progress += 1
+                  uploaded_files.append(file) if file[:success]
+
+                  if uploaded_files.size >= chunk_size or progressbar.finished?
+                    blob_ids = uploaded_files.map {|f| f['bv_id']}
+                    dirs_to_upload = dirs.clone
+                    uploaded_files = []
+                    dirs = []
+                  end
+                }
+
+                if blob_ids.present?
+                  refresh_storage_token
+                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
+
+
+                  retry_count = 0
+                  loop do
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+
+                    if not (Cnvrg::CLI.is_response_success(upload_resp, false))
+                      retry_count += 1
+                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
+                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      if retry_count > 20
+                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        break
+                      end
+                      sleep 5
+                      next
+                    end
+                    Cnvrg::Logger.info("Chunk saved on server")
+                    break
+                  end
                 end
-                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
-                uploaded_files = []
-                dirs = []
+              else
+                sleep(0.1)
               end
-            else
-              sleep(0.1)
-            end
 
-            if progressbar.finished?
-              # puts "finished"
-              file_queue.close()
-              progress_queue.close()
-              Thread.exit
+              if progressbar.finished?
+                Cnvrg::Logger.info("Progress bar finished closing queues")
+                file_queue.close()
+                progress_queue.close()
+                Thread.exit
+              end
             end
           end
         end
@@ -405,9 +433,11 @@ module Cnvrg
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
-
+          Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          next unless results
 
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
@@ -430,12 +460,11 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-
-        progress_thread.join()
+        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        progress_threads.each(&:join)
         worker_threads.each(&:join)
-
+        Thread.report_on_exception = true
       rescue => e
-        puts e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
@@ -499,7 +528,7 @@ module Cnvrg
         upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                        commit_sha1: commit_sha1, file_name: file_name,
                                                                                        file_size: file_size, file_content_type: content_type, sha1: sha1})
-        puts upload_resp
+
 
       end
 
@@ -546,7 +575,7 @@ module Cnvrg
           return false
         end
       rescue => e
-        puts e.message
+        #puts e.message
         return false
       end
     end
@@ -791,7 +820,7 @@ module Cnvrg
         return resp
 
       rescue => e
-        puts e.message
+        #puts e.message
         return false
 
       end

data/lib/cnvrg/dataset.rb

@@ -557,7 +557,12 @@ module Cnvrg
         if File.directory? file
           tree[label + "/"] = nil
         else
-          sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          begin
+            sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          rescue => e
+            #puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
+            next
+          end
           file_name = File.basename file
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)

data/lib/cnvrg/downloader/client.rb

@@ -14,7 +14,7 @@ module Cnvrg
       end
 
       def extract_key_iv(sts_path)
-        sts = open(sts_path).read rescue nil
+        sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
         raise StandardError.new("Cant open sts") if sts.blank?
         sts.split("\n")
       end

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 50, timeout: 43200)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -47,4 +47,4 @@ module Cnvrg
       end
     end
   end
-end
+end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.12'
+  VERSION = '1.10.19'
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.12
+  version: 1.10.19
 platform: ruby
 authors:
 - Yochay Ettun
 - Leah Kolben
 - Omer Shacham
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2020-09-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -438,7 +438,7 @@ files:
 homepage: https://cnvrg.io
 licenses: []
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -453,8 +453,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
-signing_key: 
+rubygems_version: 3.1.2
+signing_key:
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.
 test_files: []