cnvrg 1.10.5 → 1.10.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +20 -11
- data/lib/cnvrg/helpers.rb +8 -3
- data/lib/cnvrg/project.rb +1 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e91c3bfecec9012b07c62d288260fe11c7a650df39e06ce493f6857ccd17ea0f
|
4
|
+
data.tar.gz: cb65a865545d566b183d2829c3e76b6bacce56b741e3216b500f5ba691e529a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80e70d6faecdc53cabce9812ef6621c0dc2c72890d1122dca724544510e119b21b63f941f244097b1efe200dc6c94f83ecccc159fe891ad3a78b19c909f960ed
|
7
|
+
data.tar.gz: 9084beb5aceea84f51f1fb187e7fdb0f4c1bd3ebe2035c0969a0f6819ba36fbf77a171d0db6674edbf8cd4c5bee74d926a6a9ee74bf664f4f0c4cac283b23d0d
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1268,7 +1268,7 @@ module Cnvrg
|
|
1268
1268
|
@datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
|
1269
1269
|
|
1270
1270
|
# Init a new commit
|
1271
|
-
response = @datafiles.start_commit(false,
|
1271
|
+
response = @datafiles.start_commit(false, false, chunks: 1, message: message )
|
1272
1272
|
unless response #means we failed in the start commit.
|
1273
1273
|
raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
|
1274
1274
|
end
|
@@ -3602,16 +3602,23 @@ module Cnvrg
|
|
3602
3602
|
tries = 0
|
3603
3603
|
begin
|
3604
3604
|
result = Cnvrg::API_V2.request(
|
3605
|
-
"#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/
|
3605
|
+
"#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
|
3606
|
+
'GET',
|
3607
|
+
{ grid: res["result"]["grid"]}
|
3606
3608
|
)
|
3607
|
-
|
3608
|
-
|
3609
|
-
|
3610
|
-
break
|
3611
|
-
else
|
3609
|
+
|
3610
|
+
exit_statuses = result.values
|
3611
|
+
if exit_statuses.include? nil
|
3612
3612
|
system("clear") || system("cls")
|
3613
|
-
|
3613
|
+
msg = "#{Time.current}: waiting for experiment to finish"
|
3614
|
+
msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
|
3615
|
+
puts msg
|
3614
3616
|
sleep 3
|
3617
|
+
else
|
3618
|
+
result.each do |slug, exit_status|
|
3619
|
+
puts "Experiment #{slug} was exited with status #{exit_status}"
|
3620
|
+
end
|
3621
|
+
break
|
3615
3622
|
end
|
3616
3623
|
rescue => e
|
3617
3624
|
log_error(e)
|
@@ -4849,17 +4856,19 @@ module Cnvrg
|
|
4849
4856
|
exp_name = exp["title"]
|
4850
4857
|
if exp["end_commit"].present? and exp["status"] != "Ongoing"
|
4851
4858
|
log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
|
4852
|
-
Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
|
4859
|
+
num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
|
4853
4860
|
exps_map[exp_slug] = exp
|
4854
4861
|
else
|
4855
4862
|
log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
|
4856
|
-
success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
|
4863
|
+
success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
|
4857
4864
|
if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
|
4858
4865
|
log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
|
4859
|
-
Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
|
4866
|
+
num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
|
4860
4867
|
copied_commits << exp["last_successful_commit"]
|
4861
4868
|
end
|
4862
4869
|
end
|
4870
|
+
|
4871
|
+
log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
|
4863
4872
|
rescue => e
|
4864
4873
|
Cnvrg::Logger.log_error(e)
|
4865
4874
|
end
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -364,11 +364,14 @@ parameters:
|
|
364
364
|
@files.download_files(files, commit_sha1, progress: nil)
|
365
365
|
FileUtils.rm_rf("#{dest_dir}")
|
366
366
|
FileUtils.mkdir_p(dest_dir)
|
367
|
+
num_of_new_files = 0
|
367
368
|
files.each do |f|
|
368
369
|
file_dir = "#{dest_dir}/#{File.dirname(f)}"
|
369
370
|
FileUtils.mkdir_p(file_dir)
|
371
|
+
num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
|
370
372
|
FileUtils.mv(f, "#{dest_dir}/#{f}")
|
371
373
|
end
|
374
|
+
return num_of_new_files
|
372
375
|
end
|
373
376
|
|
374
377
|
def get_experiment_events_log_via_kubectl(exp, namespace)
|
@@ -376,7 +379,7 @@ parameters:
|
|
376
379
|
result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
|
377
380
|
|
378
381
|
pod_name = result.split(" ")[0]
|
379
|
-
return false if pod_name.blank?
|
382
|
+
return false, 0 if pod_name.blank?
|
380
383
|
FileUtils.mkdir_p(dest_dir)
|
381
384
|
working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
|
382
385
|
working_dir.strip!
|
@@ -399,16 +402,18 @@ parameters:
|
|
399
402
|
end
|
400
403
|
end
|
401
404
|
|
405
|
+
num_of_new_files = 0
|
402
406
|
all_files.each do |file|
|
403
407
|
file_dir = "#{dest_dir}/#{File.dirname(file)}"
|
404
408
|
FileUtils.mkdir_p(file_dir)
|
409
|
+
num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
|
405
410
|
res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
|
406
411
|
end
|
407
412
|
|
408
|
-
return true
|
413
|
+
return true, num_of_new_files
|
409
414
|
rescue => e
|
410
415
|
Cnvrg::Logger.log_error(e)
|
411
|
-
return false
|
416
|
+
return false, 0
|
412
417
|
end
|
413
418
|
end
|
414
419
|
|
data/lib/cnvrg/project.rb
CHANGED
@@ -421,7 +421,7 @@ module Cnvrg
|
|
421
421
|
|
422
422
|
def generate_idx(deploy: false, files: [])
|
423
423
|
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
424
|
-
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
424
|
+
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
|
425
425
|
else
|
426
426
|
old_idx = {:tree => {}, :commit => nil}
|
427
427
|
end
|
data/lib/cnvrg/version.rb
CHANGED