cnvrg 1.10.5 → 1.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 07ab3ff1cccbdc44a84d611daa11de215256e035ae6e84b624b8fc18339b20bf
-  data.tar.gz: f8ff99cae183a98a3be25222ba035e8501bf13c650842a083a9065ff238dacab
+  metadata.gz: e91c3bfecec9012b07c62d288260fe11c7a650df39e06ce493f6857ccd17ea0f
+  data.tar.gz: cb65a865545d566b183d2829c3e76b6bacce56b741e3216b500f5ba691e529a4
 SHA512:
-  metadata.gz: ab831c80f088b48ada89e56a75c732afe88482444afe758b9370a60ec1085a0375c6e2835aa28a1caefd96d26be1256f9c0a3541b64aa35c9d42ec1ac248f342
-  data.tar.gz: d3d3f35f63876d6e7730ce728f473482635c9f07bbe482e2324cbb091865caedb3ffb7405c68d5eccecc96f2e6c6ce1fa25fea4c9998b15ab851fd6dbab115f5
+  metadata.gz: 80e70d6faecdc53cabce9812ef6621c0dc2c72890d1122dca724544510e119b21b63f941f244097b1efe200dc6c94f83ecccc159fe891ad3a78b19c909f960ed
+  data.tar.gz: 9084beb5aceea84f51f1fb187e7fdb0f4c1bd3ebe2035c0969a0f6819ba36fbf77a171d0db6674edbf8cd4c5bee74d926a6a9ee74bf664f4f0c4cac283b23d0d

data/lib/cnvrg/cli.rb

@@ -1268,7 +1268,7 @@ module Cnvrg
         @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
 
         # Init a new commit
-        response = @datafiles.start_commit(false, true, chunks: 1, message: message )
+        response = @datafiles.start_commit(false, false, chunks: 1, message: message )
         unless response #means we failed in the start commit.
           raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
         end
@@ -3602,16 +3602,23 @@ module Cnvrg
               tries = 0
               begin
                 result = Cnvrg::API_V2.request(
-                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/status"
+                    "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
+                    'GET',
+                    { grid: res["result"]["grid"]}
                 )
-                exit_status = result["exit_status"]
-                if exit_status
-                  puts "Experiment was exited with status: #{exit_status}"
-                  break
-                else
+
+                exit_statuses = result.values
+                if exit_statuses.include? nil
                   system("clear") || system("cls")
-                  puts "#{Time.current}: waiting for experiment to finish"
+                  msg = "#{Time.current}: waiting for experiment to finish"
+                  msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
+                  puts msg
                   sleep 3
+                else
+                  result.each do |slug, exit_status|
+                    puts "Experiment #{slug} was exited with status #{exit_status}"
+                  end
+                  break
                 end
               rescue => e
                 log_error(e)
@@ -4849,17 +4856,19 @@ module Cnvrg
             exp_name = exp["title"]
             if exp["end_commit"].present? and exp["status"] != "Ongoing"
               log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
-              Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
+              num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
               exps_map[exp_slug] = exp
             else
               log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
-              success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
+              success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
               if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
                 log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
-                Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
+                num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
                 copied_commits << exp["last_successful_commit"]
               end
             end
+
+            log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
           rescue => e
             Cnvrg::Logger.log_error(e)
           end

data/lib/cnvrg/helpers.rb

@@ -364,11 +364,14 @@ parameters:
       @files.download_files(files, commit_sha1, progress: nil)
       FileUtils.rm_rf("#{dest_dir}")
       FileUtils.mkdir_p(dest_dir)
+      num_of_new_files = 0
       files.each do |f|
         file_dir = "#{dest_dir}/#{File.dirname(f)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
         FileUtils.mv(f, "#{dest_dir}/#{f}")
       end
+      return num_of_new_files
     end
 
     def get_experiment_events_log_via_kubectl(exp, namespace)
@@ -376,7 +379,7 @@ parameters:
       result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
 
       pod_name = result.split(" ")[0]
-      return false if pod_name.blank?
+      return false, 0 if pod_name.blank?
       FileUtils.mkdir_p(dest_dir)
       working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
       working_dir.strip!
@@ -399,16 +402,18 @@ parameters:
         end
       end
 
+      num_of_new_files = 0
       all_files.each do |file|
         file_dir = "#{dest_dir}/#{File.dirname(file)}"
         FileUtils.mkdir_p(file_dir)
+        num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
         res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
       end
 
-      return true
+      return true, num_of_new_files
     rescue => e
       Cnvrg::Logger.log_error(e)
-      return false
+      return false, 0
     end
   end
 

data/lib/cnvrg/project.rb

@@ -421,7 +421,7 @@ module Cnvrg
 
     def generate_idx(deploy: false, files: [])
       if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
-        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
+        old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
       else
         old_idx = {:tree => {}, :commit => nil}
       end

data/lib/cnvrg/version.rb

@@ -1,3 +1,3 @@
 module Cnvrg
-  VERSION = '1.10.5'
+  VERSION = '1.10.6'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.10.5
+  version: 1.10.6
 platform: ruby
 authors:
 - Yochay Ettun