cnvrg 1.10.7 → 1.10.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1232c4c81a6429fdf01484ba321224d3e9a03c19914a59575eb867d12b206bdd
-  data.tar.gz: 21ab04f21f6850fc6c9e7eddbcf807b6f7ac312f3c00a857be24cf38d710a784
+  metadata.gz: 29b4ab96b94066d5e799cbd5e648eaae0871a634ac6383a66ca3c82a3af9a038
+  data.tar.gz: aca3f52a3959cf6db6614bed5b7669c41e9607f0abf25ec83e870783c997f484
 SHA512:
-  metadata.gz: 27c2738e285408a315fa8384b230d7c3e108e0b98ecf6b6bb57ee5d7996cbcce37db94fb8b98846195535ac24d7539a7ccfb72ef94fdc82955de1cc2e6e60300
-  data.tar.gz: 470f1e94d6bbba179a637fdd9765c9b052f91bf1c37742d56541f592cb5a4fd38c11ffd12d554ed6d38b1a0d6f3eeacef84f3032afc175fc6a041880578c9e6e
+  metadata.gz: 96fbd5de39c5aa44d93e24c529b5e149cbd8f689c56329e7c5b4a6f9990d19ca8a7d1fdb1f5f00b8abc05c8dccc061a74de65ac8c81524c27f65b0e349dfdc23
+  data.tar.gz: 2412c51b6926725be234c022faca6191110caf45de2c83381b171e411f7d7bee78e74d519b91f42e01d07b69413df5df009747f35bd34e480d5733718ccd824c

data/lib/cnvrg/cli.rb

@@ -1208,12 +1208,14 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
+          Cnvrg::Logger.info("Creating commit")
           response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
           @commit =  response['result']['commit_sha1']
         elsif commit.eql? "latest"
+          Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
@@ -1233,15 +1235,15 @@ module Cnvrg
           override: override,
           chunk_size: chunk_size,
           prefix: dir,
-          threads: threads
+          threads: threads,
         )
-
+        Cnvrg::Logger.info("Finished upload files")
         # This is for backwards compatibility only and should be removed in future versions:
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
           raise SignalException.new(1, res.msg)
         end
-
+        Cnvrg::Logger.info("Saving commit on server")
         res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
@@ -1273,6 +1275,17 @@ module Cnvrg
           raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
         end
         @commit =  response['result']['commit_sha1']
+
+        # Server expects certain regex format with * so fix those that dont comply
+        regex_list = regex_list.map do |regex|
+          if regex.end_with? "/"
+            # if user wants to delete entire folder add regex to delete contents as well
+            [regex, "#{regex}*"]
+          else
+            regex
+          end
+        end.flatten
+
         files_to_delete, folders_to_delete, job_id = @datafiles.delete_multiple_files(@commit, regex_list)
         log_message("Deleting #{files_to_delete} files and #{folders_to_delete} folders", Thor::Shell::Color::GREEN)
 
@@ -3083,6 +3096,11 @@ module Cnvrg
       emails = options["emails"]
       wait = options["wait"]
 
+      if wait && local
+        log_message("WARN: `wait` option is not valid for local experiment, ignoring it", Thor::Shell::Color::YELLOW)
+        wait = false
+      end
+
       if !data.present? and data_query.present?
         log_message("Please provide data with data_query", Thor::Shell::Color::RED)
         exit(1)
@@ -3598,24 +3616,33 @@ module Cnvrg
           exit_status = 0
           
           if options['wait']
+            end_pos = 0
             while true
               tries = 0
               begin
                 result = Cnvrg::API_V2.request(
-                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
+                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/info",
                     'GET',
-                    { grid: res["result"]["grid"]}
+                    { exit_status: true, grid: res["result"]["grid"], pos: end_pos }
                 )
 
-                exit_statuses = result.values
+                exit_statuses = result.values.pluck('exit_status')
                 if exit_statuses.include? nil
-                  system("clear") || system("cls")
-                  msg = "#{Time.current}: waiting for experiment to finish"
-                  msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
-                  puts msg
+                  if res["result"]["grid"]
+                    system("clear") || system("cls")
+                    msg = "#{Time.current}: waiting for all experiments to finish"
+                    puts msg
+                  else
+                    end_pos = result[res['result']['exp_url']]['end_pos']
+                    logs = result[res['result']['exp_url']]['logs']
+                    logs.each do |log|
+                      puts log['message']
+                    end
+                  end
                   sleep 3
                 else
-                  result.each do |slug, exit_status|
+                  result.each do |slug, value|
+                    exit_status = value['exit_status']
                     puts "Experiment #{slug} was exited with status #{exit_status}"
                   end
                   break
@@ -5282,8 +5309,8 @@ module Cnvrg
               count += 1
             end
             if File.exist? logfile_old
-              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              #@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              #@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
               FileUtils.remove logfile_old
             end
 

data/lib/cnvrg/data.rb

@@ -214,7 +214,7 @@ module Cnvrg
 
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    # method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
@@ -226,20 +226,21 @@ module Cnvrg
       dir = options[:dir]
       force = options[:force]
       override = options[:override]
-      commit = options[:commit]
+      # commit = options[:commit]
+      commit = ''
       message = options[:message]
       threads = options[:threads]
       chunk_size = options[:chunk_size]
       cli.data_put(
-          dataset_url,
-          files: files,
-          dir: dir,
-          commit: commit,
-          force: force,
-          override: override,
-          threads: threads,
-          chunk_size: chunk_size,
-          message: message
+        dataset_url,
+        files: files,
+        dir: dir,
+        commit: commit,
+        force: force,
+        override: override,
+        threads: threads,
+        chunk_size: chunk_size,
+        message: message
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -376,16 +376,16 @@ module Cnvrg
 
               if uploaded_files.size == chunk_size or progressbar.finished?
                 refresh_storage_token
-                # puts "progress: #{progress_queue.length}"
-                # puts "files: #{file_queue.length}"
+                Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files")
                 Cnvrg::Logger.info("Sending Upload files save")
                 blob_ids = uploaded_files.map {|f| f['bv_id']}
+                Cnvrg::Logger.info("Sending chunk to server")
                 upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
                 unless Cnvrg::CLI.is_response_success(upload_resp, false)
                   Cnvrg::Logger.log_method(bind: binding)
                   raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
                 end
-                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
+                Cnvrg::Logger.info("Chunk saved on server")
                 uploaded_files = []
                 dirs = []
               end
@@ -394,7 +394,7 @@ module Cnvrg
             end
 
             if progressbar.finished?
-              # puts "finished"
+              Cnvrg::Logger.info("Progress bar finished closing queues")
               file_queue.close()
               progress_queue.close()
               Thread.exit
@@ -405,12 +405,11 @@ module Cnvrg
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
-
+          Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
 
-          # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
-
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
@@ -432,7 +431,7 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-
+        Cnvrg::Logger.info("Finishing sub processes of datasets' upload")
         progress_thread.join()
         worker_threads.each(&:join)
 

data/lib/cnvrg/dataset.rb

@@ -544,8 +544,12 @@ module Cnvrg
     def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
-        #check if prefix exists do prefix/path otherwise path
-        label = file.gsub(self.local_path + "/", "")
+
+        # Fix for root path issue
+        safe_path = file
+        safe_path = file[1..-1] if file.start_with? "/"
+
+        label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           raise StandardError.new("#{label} is not a valid file name.")
@@ -558,9 +562,16 @@ module Cnvrg
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)
           content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-          relative_path = file.gsub(/^#{@local_path + "/"}/, "")
+          relative_path = safe_path.gsub(/^#{@local_path + "/"}/, "")
           relative_path = "#{prefix}/#{relative_path}" if prefix.present?
-          tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path}
+          tree[label] = {
+            sha1: sha1,
+            file_name: file_name,
+            file_size: file_size,
+            content_type: content_type,
+            absolute_path: file,
+            relative_path: relative_path
+          }
         end
       end
       if prefix.present? #add the prefix as dirs to the files
@@ -873,13 +884,8 @@ module Cnvrg
     end
 
     def self.stop_if_dataset_present(dataset_home, dataset_name, commit: nil)
-
       cli = Cnvrg::CLI.new()
       config = YAML.load_file(dataset_home + "/.cnvrg/config.yml")
-      if commit.present?
-        local_commit = YAML.load_file(dataset_home + "/.cnvrg/idx.yml")[:commit] rescue nil
-        return if commit != local_commit or local_commit.blank?
-      end
       if config[:dataset_name] == dataset_name
         cli.log_message("Dataset already present, clone aborted")
         exit(0)
@@ -887,6 +893,5 @@ module Cnvrg
     rescue => e
       nil
     end
-
   end
 end

data/lib/cnvrg/downloader/clients/gcp_client.rb

@@ -11,7 +11,7 @@ module Cnvrg
           @tempfile = nil
           @bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
           init_gcp_credentials
-          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20)
+          @storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 50, timeout: 43200)
           @bucket = @storage.bucket(@bucket_name)
           @bucket.name
         rescue => e
@@ -47,4 +47,4 @@ module Cnvrg
       end
     end
   end
-end
+end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.7'
+  VERSION = '1.10.17'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.7
+  version: 1.10.17
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-16 00:00:00.000000000 Z
+date: 2020-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A CLI tool for interacting with cnvrg.io.