cnvrg 1.10.8 → 1.10.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d9e8e76336bf9862fca95649740a7b5a5148565469ece725f4760041db28ffc4
-  data.tar.gz: f65852f97fb1294258c3c6b4b6941fb62248eeb1691d432e47bb4e2acb60a313
+  metadata.gz: 8009827a102d9ea4bda3e07d95ad52a23fe7202b2670850952fb345acac7b906
+  data.tar.gz: aa91b22c83120d50f0631945d06f4d878e1cea4d2f9d950f6e29fbba17305f21
 SHA512:
-  metadata.gz: c5be8e6c872dd256560805d8146b3e2d7672a72844b932081dddc94c62e3fde4f9d7f66dd18d4f882c005c9e6e2183a91de689254b6dce07c265932058a4a1b0
-  data.tar.gz: 345eee06f18b24a30826665c7803b1275bdbc33d94779e5e37812cce7f95738bb819c5e2064871618145a27e72d133373e2493fb57f63b1c85ce8072c6057a54
+  metadata.gz: 2159ce3f1e3b55390b599a72b80c1b6de559554380d5b294922ddd001cff2c944cf6614bb582f184c62be6299826f4a1860257a5426c6534a254ea195bb9910c
+  data.tar.gz: ccb989d604602dbcfcee6f40725ae8a45875744f222b7aa717ddfe2760483361d03a16f6e016179acf82c0c493d9a3518a548d18dbb850a7cf0fc0b99aef0120

data/lib/cnvrg/cli.rb

@@ -1208,12 +1208,14 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
+          Cnvrg::Logger.info("Creating commit")
           response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
           @commit =  response['result']['commit_sha1']
         elsif commit.eql? "latest"
+          Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
@@ -1233,15 +1235,15 @@ module Cnvrg
           override: override,
           chunk_size: chunk_size,
           prefix: dir,
-          threads: threads
+          threads: threads,
         )
-
+        Cnvrg::Logger.info("Finished upload files")
         # This is for backwards compatibility only and should be removed in future versions:
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
           raise SignalException.new(1, res.msg)
         end
-
+        Cnvrg::Logger.info("Saving commit on server")
         res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
@@ -1273,6 +1275,17 @@ module Cnvrg
           raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
         end
         @commit =  response['result']['commit_sha1']
+
+        # Server expects certain regex format with * so fix those that dont comply
+        regex_list = regex_list.map do |regex|
+          if regex.end_with? "/"
+            # if user wants to delete entire folder add regex to delete contents as well
+            [regex, "#{regex}*"]
+          else
+            regex
+          end
+        end.flatten
+
         files_to_delete, folders_to_delete, job_id = @datafiles.delete_multiple_files(@commit, regex_list)
         log_message("Deleting #{files_to_delete} files and #{folders_to_delete} folders", Thor::Shell::Color::GREEN)
 
@@ -3083,6 +3096,11 @@ module Cnvrg
       emails = options["emails"]
       wait = options["wait"]
 
+      if wait && local
+        log_message("WARN: `wait` option is not valid for local experiment, ignoring it", Thor::Shell::Color::YELLOW)
+        wait = false
+      end
+
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
         exit(1)
@@ -3598,24 +3616,33 @@ module Cnvrg
           exit_status = 0
           
           if options['wait']
+            end_pos = 0
             while true
               tries = 0
               begin
                 result = Cnvrg::API_V2.request(
-                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
+                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/info",
                     'GET',
-                    { grid: res["result"]["grid"]}
+                    { exit_status: true, grid: res["result"]["grid"], pos: end_pos }
                 )
 
-                exit_statuses = result.values
+                exit_statuses = result.values.pluck('exit_status')
                 if exit_statuses.include? nil
-                  system("clear") || system("cls")
-                  msg = "#{Time.current}: waiting for experiment to finish"
-                  msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
-                  puts msg
+                  if res["result"]["grid"]
+                    system("clear") || system("cls")
+                    msg = "#{Time.current}: waiting for all experiments to finish"
+                    puts msg
+                  else
+                    end_pos = result[res['result']['exp_url']]['end_pos']
+                    logs = result[res['result']['exp_url']]['logs']
+                    logs.each do |log|
+                      puts log['message']
+                    end
+                  end
                   sleep 3
                 else
-                  result.each do |slug, exit_status|
+                  result.each do |slug, value|
+                    exit_status = value['exit_status']
                     puts "Experiment #{slug} was exited with status #{exit_status}"
                   end
                   break
@@ -5282,8 +5309,8 @@ module Cnvrg
               count += 1
             end
             if File.exist? logfile_old
-              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              #@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              #@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
               FileUtils.remove logfile_old
             end
 

data/lib/cnvrg/data.rb

@@ -214,7 +214,7 @@ module Cnvrg
 
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    # method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
@@ -226,20 +226,21 @@ module Cnvrg
       dir = options[:dir]
       force = options[:force]
       override = options[:override]
-      commit = options[:commit]
+      # commit = options[:commit]
+      commit = ''
       message = options[:message]
       threads = options[:threads]
       chunk_size = options[:chunk_size]
       cli.data_put(
-          dataset_url,
-          files: files,
-          dir: dir,
-          commit: commit,
-          force: force,
-          override: override,
-          threads: threads,
-          chunk_size: chunk_size,
-          message: message
+        dataset_url,
+        files: files,
+        dir: dir,
+        commit: commit,
+        force: force,
+        override: override,
+        threads: threads,
+        chunk_size: chunk_size,
+        message: message
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -316,21 +316,21 @@ module Cnvrg
           #Cnvrg::Logger.log_method(bind: binding)
           retry_count += 1
 
-          puts "Failed request upload files: #{Time.current}"
-          puts upload_resp
+          Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
 
-          if retry_count > 5
-            raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+          if retry_count > 20
+            puts "Failed to upload files: #{Time.current}, trying next chunk"
+            return nil
           end
           sleep 5
           next
         end
         return upload_resp['result'].with_indifferent_access
       end
-
     end
 
     def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
+      Thread.report_on_exception = false
       cli = CLI.new
       cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
 
@@ -342,6 +342,7 @@ module Cnvrg
       file_queue = Queue.new
       progress_queue = Queue.new
       worker_threads = []
+      progress_threads = []
 
       # Vars to keep track of uploaded files and directories
       dirs = []
@@ -359,45 +360,69 @@ module Cnvrg
                 cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
                 Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
               end
+              while progress_queue.size > 15000
+                sleep(0.1)
+              end
               progress_queue << file
             end
           end
         end
 
         # init the thread that handles the file upload progress and saving them in the server
-        progress_thread = Thread.new do
-          loop do
-            file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
-            unless file.nil?
-              progress_mutex.synchronize {
-                progressbar.progress += 1
-                uploaded_files.append(file) if file[:success]
-              }
-
-              if uploaded_files.size == chunk_size or progressbar.finished?
-                refresh_storage_token
-                # puts "progress: #{progress_queue.length}"
-                # puts "files: #{file_queue.length}"
-                Cnvrg::Logger.info("Sending Upload files save")
-                blob_ids = uploaded_files.map {|f| f['bv_id']}
-                upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
-                unless Cnvrg::CLI.is_response_success(upload_resp, false)
-                  Cnvrg::Logger.log_method(bind: binding)
-                  raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+        threads.times do |i|
+          progress_threads[i] = Thread.new do
+            loop do
+              file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
+              unless file.nil?
+                blob_ids = []
+                dirs_to_upload = []
+
+                progress_mutex.synchronize {
+                  progressbar.progress += 1
+                  uploaded_files.append(file) if file[:success]
+
+                  if uploaded_files.size >= chunk_size or progressbar.finished?
+                    blob_ids = uploaded_files.map {|f| f['bv_id']}
+                    dirs_to_upload = dirs.clone
+                    uploaded_files = []
+                    dirs = []
+                  end
+                }
+
+                if blob_ids.present?
+                  refresh_storage_token
+                  Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
+
+
+                  retry_count = 0
+                  loop do
+                    upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
+
+                    if not (Cnvrg::CLI.is_response_success(upload_resp, false))
+                      retry_count += 1
+                      Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
+                      Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
+                      if retry_count > 20
+                        puts "Failed to save files: #{Time.current}, trying next chunk"
+                        break
+                      end
+                      sleep 5
+                      next
+                    end
+                    Cnvrg::Logger.info("Chunk saved on server")
+                    break
+                  end
                 end
-                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
-                uploaded_files = []
-                dirs = []
+              else
+                sleep(0.1)
               end
-            else
-              sleep(0.1)
-            end
 
-            if progressbar.finished?
-              # puts "finished"
-              file_queue.close()
-              progress_queue.close()
-              Thread.exit
+              if progressbar.finished?
+                Cnvrg::Logger.info("Progress bar finished closing queues")
+                file_queue.close()
+                progress_queue.close()
+                Thread.exit
+              end
             end
           end
         end
@@ -405,9 +430,11 @@ module Cnvrg
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
-
+          Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
+          next unless results
 
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
@@ -430,12 +457,11 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-
-        progress_thread.join()
+        Cnvrg::Logger.info("Waiting to progress and workers to finish")
+        progress_threads.each(&:join)
         worker_threads.each(&:join)
-
+        Thread.report_on_exception = true
       rescue => e
-        puts e
         Cnvrg::Logger.log_method(bind: binding)
         Cnvrg::Logger.log_error(e)
         raise e
@@ -499,7 +525,7 @@ module Cnvrg
         upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
                                                                                        commit_sha1: commit_sha1, file_name: file_name,
                                                                                        file_size: file_size, file_content_type: content_type, sha1: sha1})
-        puts upload_resp
+
 
       end
 
@@ -546,7 +572,7 @@ module Cnvrg
           return false
         end
       rescue => e
-        puts e.message
+        #puts e.message
         return false
       end
     end
@@ -791,7 +817,7 @@ module Cnvrg
         return resp
 
       rescue => e
-        puts e.message
+        #puts e.message
         return false
 
       end

data/lib/cnvrg/dataset.rb

@@ -557,7 +557,12 @@ module Cnvrg
         if File.directory? file
           tree[label + "/"] = nil
         else
-          sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          begin
+            sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
+          rescue => e
+            #puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
+            next
+          end
           file_name = File.basename file
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)
@@ -884,13 +889,8 @@ module Cnvrg
     end
 
     def self.stop_if_dataset_present(dataset_home, dataset_name, commit: nil)
-
       cli = Cnvrg::CLI.new()
       config = YAML.load_file(dataset_home + "/.cnvrg/config.yml")
-      if commit.present?
-        local_commit = YAML.load_file(dataset_home + "/.cnvrg/idx.yml")[:commit] rescue nil
-        return if commit != local_commit or local_commit.blank?
-      end
       if config[:dataset_name] == dataset_name
         cli.log_message("Dataset already present, clone aborted")
         exit(0)
@@ -898,6 +898,5 @@ module Cnvrg
     rescue => e
       nil
     end
-
   end
 end

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 50, timeout: 43200)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -47,4 +47,4 @@ module Cnvrg
       end
     end
   end
-end
+end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.8'
+  VERSION = '1.10.18'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.8
+  version: 1.10.18
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-20 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler