cnvrg 1.10.5 → 1.10.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07ab3ff1cccbdc44a84d611daa11de215256e035ae6e84b624b8fc18339b20bf
-  data.tar.gz: f8ff99cae183a98a3be25222ba035e8501bf13c650842a083a9065ff238dacab
+  metadata.gz: 4d377618af4c931ec6553e946bb1f32c25bbb7bccd538f040309f4318c20e3bd
+  data.tar.gz: 744a798d06c8c562991be9b68cf6b8258870e657a93e2125ff3bc3c76bd7b2b6
 SHA512:
-  metadata.gz: ab831c80f088b48ada89e56a75c732afe88482444afe758b9370a60ec1085a0375c6e2835aa28a1caefd96d26be1256f9c0a3541b64aa35c9d42ec1ac248f342
-  data.tar.gz: d3d3f35f63876d6e7730ce728f473482635c9f07bbe482e2324cbb091865caedb3ffb7405c68d5eccecc96f2e6c6ce1fa25fea4c9998b15ab851fd6dbab115f5
+  metadata.gz: e52939b069a7ef45d69a0ae8a086291245db90602a13039c7981ac9a3c85913cf3c3b105198a149cc2913668a5fcf59288266ada60d71b5c327b3b90bc3c594a
+  data.tar.gz: 03d8880d00c707ab6afcb3509da1e02c4f01aeaf5e378b49a3ec7c651b2b00176b01be5de9320da652173480279bf1d820cf9e2a255ec1e21217e361c65fd375

data/lib/cnvrg/cli.rb

@@ -1208,12 +1208,14 @@ module Cnvrg
         log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
         number_of_chunks = (@files.size.to_f / chunk_size).ceil
         if commit.blank?
+          Cnvrg::Logger.info("Creating commit")
           response = @datafiles.start_commit(false, force, chunks: number_of_chunks, message: message )
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
           end
           @commit =  response['result']['commit_sha1']
         elsif commit.eql? "latest"
+          Cnvrg::Logger.info("Put files in latest commit")
           response = @datafiles.last_valid_commit()
           unless response #means we failed in the start commit.
             raise SignalException.new(1, "Cant put files into commit:#{commit}, check the dataset id and commitc")
@@ -1233,15 +1235,15 @@ module Cnvrg
           override: override,
           chunk_size: chunk_size,
           prefix: dir,
-          threads: threads
+          threads: threads,
         )
-
+        Cnvrg::Logger.info("Finished upload files")
         # This is for backwards compatibility only and should be removed in future versions:
         res = @datafiles.put_commit(@commit)
         unless res.is_success?
           raise SignalException.new(1, res.msg)
         end
-
+        Cnvrg::Logger.info("Saving commit on server")
         res = @datafiles.end_commit(@commit,force, success: true, commit_type: "put")
         msg = res['result']
         response = Cnvrg::Result.new(Cnvrg::CLI.is_response_success(res, true), msg)
@@ -1268,11 +1270,22 @@ module Cnvrg
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
 
         # Init a new commit
-        response = @datafiles.start_commit(false, true, chunks: 1, message: message )
+        response = @datafiles.start_commit(false, false, chunks: 1, message: message )
         unless response #means we failed in the start commit.
           raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
         end
         @commit =  response['result']['commit_sha1']
+
+        # Server expects certain regex format with * so fix those that dont comply
+        regex_list = regex_list.map do |regex|
+          if regex.end_with? "/"
+            # if user wants to delete entire folder add regex to delete contents as well
+            [regex, "#{regex}*"]
+          else
+            regex
+          end
+        end.flatten
+
         files_to_delete, folders_to_delete, job_id = @datafiles.delete_multiple_files(@commit, regex_list)
         log_message("Deleting #{files_to_delete} files and #{folders_to_delete} folders", Thor::Shell::Color::GREEN)
 
@@ -3598,20 +3611,35 @@ module Cnvrg
           exit_status = 0
           
           if options['wait']
+            end_pos = 0
             while true
               tries = 0
               begin
                 result = Cnvrg::API_V2.request(
-                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/status"
+                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/info",
+                    'GET',
+                    { exit_status: true, grid: res["result"]["grid"], pos: end_pos }
                 )
-                exit_status = result["exit_status"]
-                if exit_status
-                  puts "Experiment was exited with status: #{exit_status}"
-                  break
-                else
-                  system("clear") || system("cls")
-                  puts "#{Time.current}: waiting for experiment to finish"
+
+                exit_statuses = result.values.pluck('exit_status')
+                if exit_statuses.include? nil
+                  if res["result"]["grid"]
+                    system("clear") || system("cls")
+                    msg = "#{Time.current}: waiting for all experiments to finish"
+                    puts msg
+                  else
+                    end_pos = result[res['result']['exp_url']]['end_pos']
+                    logs = result[res['result']['exp_url']]['logs']
+                    logs.each do |log|
+                      puts log['message']
+                    end
+                  end
                   sleep 3
+                else
+                  result.each do |slug, value|
+                    puts "Experiment #{slug} was exited with status #{value['exit_status']}"
+                  end
+                  break
                 end
               rescue => e
                 log_error(e)
@@ -4849,17 +4877,19 @@ module Cnvrg
             exp_name = exp["title"]
             if exp["end_commit"].present? and exp["status"] != "Ongoing"
               log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
-              Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
+              num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
               exps_map[exp_slug] = exp
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
-              success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+              success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
               if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
                 copied_commits << exp["last_successful_commit"]
               end
             end
+
+            log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
           rescue => e
             Cnvrg::Logger.log_error(e)
           end
@@ -5273,8 +5303,8 @@ module Cnvrg
               count += 1
             end
             if File.exist? logfile_old
-              @files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
-              @files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
+              #@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
+              #@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
               FileUtils.remove logfile_old
             end
 

data/lib/cnvrg/data.rb

@@ -214,7 +214,7 @@ module Cnvrg
 
     desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
     method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
-    method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
+    # method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
     method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
     method_option :override, :type => :boolean, :aliases => ["--override"], :default => false
     method_option :threads, :type => :numeric, :aliases => ["-t","--threads"], :default => 15
@@ -226,20 +226,21 @@ module Cnvrg
       dir = options[:dir]
       force = options[:force]
       override = options[:override]
-      commit = options[:commit]
+      # commit = options[:commit]
+      commit = ''
       message = options[:message]
       threads = options[:threads]
       chunk_size = options[:chunk_size]
       cli.data_put(
-          dataset_url,
-          files: files,
-          dir: dir,
-          commit: commit,
-          force: force,
-          override: override,
-          threads: threads,
-          chunk_size: chunk_size,
-          message: message
+        dataset_url,
+        files: files,
+        dir: dir,
+        commit: commit,
+        force: force,
+        override: override,
+        threads: threads,
+        chunk_size: chunk_size,
+        message: message
       )
     end
 

data/lib/cnvrg/datafiles.rb

@@ -376,16 +376,16 @@ module Cnvrg
 
               if uploaded_files.size == chunk_size or progressbar.finished?
                 refresh_storage_token
-                # puts "progress: #{progress_queue.length}"
-                # puts "files: #{file_queue.length}"
+                Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files")
                 Cnvrg::Logger.info("Sending Upload files save")
                 blob_ids = uploaded_files.map {|f| f['bv_id']}
+                Cnvrg::Logger.info("Sending chunk to server")
                 upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
                 unless Cnvrg::CLI.is_response_success(upload_resp, false)
                   Cnvrg::Logger.log_method(bind: binding)
                   raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
                 end
-                # cli.log_message("Saved file chunk to server", Thor::Shell::Color::GREEN)
+                Cnvrg::Logger.info("Chunk saved on server")
                 uploaded_files = []
                 dirs = []
               end
@@ -394,7 +394,7 @@ module Cnvrg
             end
 
             if progressbar.finished?
-              # puts "finished"
+              Cnvrg::Logger.info("Progress bar finished closing queues")
               file_queue.close()
               progress_queue.close()
               Thread.exit
@@ -405,12 +405,11 @@ module Cnvrg
         file_chunks = files.each_slice(chunk_size).to_a
         # Fetch the required files from the server:
         Parallel.map((file_chunks), in_threads: 10) do |files_chunk|
-
+          Cnvrg::Logger.info("Generating chunk idx")
           tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
+          Cnvrg::Logger.info("Getting files info from server")
           results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
 
-          # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
-
           if results['files'].blank?
             progress_mutex.synchronize { progressbar.progress += tree.keys.length }
             next
@@ -432,7 +431,7 @@ module Cnvrg
             file_queue.push tree[key].merge(files_to_upload[key])
           end
         end
-
+        Cnvrg::Logger.info("Finishing sub processes of datasets' upload")
         progress_thread.join()
         worker_threads.each(&:join)
 

data/lib/cnvrg/dataset.rb

@@ -544,8 +544,12 @@ module Cnvrg
     def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
       tree = {}
       Parallel.map(list_files, in_threads: threads) do |file|
-        #check if prefix exists do prefix/path otherwise path
-        label = file.gsub(self.local_path + "/", "")
+
+        # Fix for root path issue
+        safe_path = file
+        safe_path = file[1..-1] if file.start_with? "/"
+
+        label = safe_path.gsub(self.local_path + "/", "")
         label = "#{prefix}/#{label}" if prefix.present?
         if not Cnvrg::Files.valid_file_name?(label)
           raise StandardError.new("#{label} is not a valid file name.")
@@ -558,9 +562,16 @@ module Cnvrg
           file_size = File.size(file).to_f
           mime_type = MimeMagic.by_path(file)
           content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
-          relative_path = file.gsub(/^#{@local_path + "/"}/, "")
+          relative_path = safe_path.gsub(/^#{@local_path + "/"}/, "")
           relative_path = "#{prefix}/#{relative_path}" if prefix.present?
-          tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path}
+          tree[label] = {
+            sha1: sha1,
+            file_name: file_name,
+            file_size: file_size,
+            content_type: content_type,
+            absolute_path: file,
+            relative_path: relative_path
+          }
         end
       end
       if prefix.present? #add the prefix as dirs to the files
@@ -873,13 +884,8 @@ module Cnvrg
     end
 
     def self.stop_if_dataset_present(dataset_home, dataset_name, commit: nil)
-
       cli = Cnvrg::CLI.new()
       config = YAML.load_file(dataset_home + "/.cnvrg/config.yml")
-      if commit.present?
-        local_commit = YAML.load_file(dataset_home + "/.cnvrg/idx.yml")[:commit] rescue nil
-        return if commit != local_commit or local_commit.blank?
-      end
       if config[:dataset_name] == dataset_name
         cli.log_message("Dataset already present, clone aborted")
         exit(0)
@@ -887,6 +893,5 @@ module Cnvrg
     rescue => e
       nil
     end
-
   end
 end

data/lib/cnvrg/helpers.rb

@@ -364,11 +364,14 @@ parameters:
       @files.download_files(files, commit_sha1, progress: nil)
       FileUtils.rm_rf("#{dest_dir}")
       FileUtils.mkdir_p(dest_dir)
+      num_of_new_files = 0
       files.each do |f|
         file_dir = "#{dest_dir}/#{File.dirname(f)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
         FileUtils.mv(f, "#{dest_dir}/#{f}")
       end
+      return num_of_new_files
     end
 
     def get_experiment_events_log_via_kubectl(exp, namespace)
@@ -376,7 +379,7 @@ parameters:
       result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
 
       pod_name = result.split(" ")[0]
-      return false if pod_name.blank?
+      return false, 0 if pod_name.blank?
       FileUtils.mkdir_p(dest_dir)
       working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
       working_dir.strip!
@@ -399,16 +402,18 @@ parameters:
         end
       end
 
+      num_of_new_files = 0
       all_files.each do |file|
         file_dir = "#{dest_dir}/#{File.dirname(file)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
         res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
       end
 
-      return true
+      return true, num_of_new_files
     rescue => e
       Cnvrg::Logger.log_error(e)
-      return false
+      return false, 0
     end
   end
 

data/lib/cnvrg/project.rb

@@ -421,7 +421,7 @@ module Cnvrg
 
     def generate_idx(deploy: false, files: [])
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
       else
         old_idx = {:tree => {}, :commit => nil}
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.5'
+  VERSION = '1.10.15'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.5
+  version: 1.10.15
 platform: ruby
 authors:
 - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-16 00:00:00.000000000 Z
+date: 2020-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler