cnvrg 1.10.5 → 1.10.6

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 07ab3ff1cccbdc44a84d611daa11de215256e035ae6e84b624b8fc18339b20bf
4
- data.tar.gz: f8ff99cae183a98a3be25222ba035e8501bf13c650842a083a9065ff238dacab
3
+ metadata.gz: e91c3bfecec9012b07c62d288260fe11c7a650df39e06ce493f6857ccd17ea0f
4
+ data.tar.gz: cb65a865545d566b183d2829c3e76b6bacce56b741e3216b500f5ba691e529a4
5
5
  SHA512:
6
- metadata.gz: ab831c80f088b48ada89e56a75c732afe88482444afe758b9370a60ec1085a0375c6e2835aa28a1caefd96d26be1256f9c0a3541b64aa35c9d42ec1ac248f342
7
- data.tar.gz: d3d3f35f63876d6e7730ce728f473482635c9f07bbe482e2324cbb091865caedb3ffb7405c68d5eccecc96f2e6c6ce1fa25fea4c9998b15ab851fd6dbab115f5
6
+ metadata.gz: 80e70d6faecdc53cabce9812ef6621c0dc2c72890d1122dca724544510e119b21b63f941f244097b1efe200dc6c94f83ecccc159fe891ad3a78b19c909f960ed
7
+ data.tar.gz: 9084beb5aceea84f51f1fb187e7fdb0f4c1bd3ebe2035c0969a0f6819ba36fbf77a171d0db6674edbf8cd4c5bee74d926a6a9ee74bf664f4f0c4cac283b23d0d
@@ -1268,7 +1268,7 @@ module Cnvrg
1268
1268
  @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
1269
1269
 
1270
1270
  # Init a new commit
1271
- response = @datafiles.start_commit(false, true, chunks: 1, message: message )
1271
+ response = @datafiles.start_commit(false, false, chunks: 1, message: message )
1272
1272
  unless response #means we failed in the start commit.
1273
1273
  raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
1274
1274
  end
@@ -3602,16 +3602,23 @@ module Cnvrg
3602
3602
  tries = 0
3603
3603
  begin
3604
3604
  result = Cnvrg::API_V2.request(
3605
- "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/status"
3605
+ "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
3606
+ 'GET',
3607
+ { grid: res["result"]["grid"]}
3606
3608
  )
3607
- exit_status = result["exit_status"]
3608
- if exit_status
3609
- puts "Experiment was exited with status: #{exit_status}"
3610
- break
3611
- else
3609
+
3610
+ exit_statuses = result.values
3611
+ if exit_statuses.include? nil
3612
3612
  system("clear") || system("cls")
3613
- puts "#{Time.current}: waiting for experiment to finish"
3613
+ msg = "#{Time.current}: waiting for experiment to finish"
3614
+ msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
3615
+ puts msg
3614
3616
  sleep 3
3617
+ else
3618
+ result.each do |slug, exit_status|
3619
+ puts "Experiment #{slug} was exited with status #{exit_status}"
3620
+ end
3621
+ break
3615
3622
  end
3616
3623
  rescue => e
3617
3624
  log_error(e)
@@ -4849,17 +4856,19 @@ module Cnvrg
4849
4856
  exp_name = exp["title"]
4850
4857
  if exp["end_commit"].present? and exp["status"] != "Ongoing"
4851
4858
  log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
4852
- Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
4859
+ num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
4853
4860
  exps_map[exp_slug] = exp
4854
4861
  else
4855
4862
  log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
4856
- success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
4863
+ success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
4857
4864
  if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
4858
4865
  log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
4859
- Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
4866
+ num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
4860
4867
  copied_commits << exp["last_successful_commit"]
4861
4868
  end
4862
4869
  end
4870
+
4871
+ log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
4863
4872
  rescue => e
4864
4873
  Cnvrg::Logger.log_error(e)
4865
4874
  end
@@ -364,11 +364,14 @@ parameters:
364
364
  @files.download_files(files, commit_sha1, progress: nil)
365
365
  FileUtils.rm_rf("#{dest_dir}")
366
366
  FileUtils.mkdir_p(dest_dir)
367
+ num_of_new_files = 0
367
368
  files.each do |f|
368
369
  file_dir = "#{dest_dir}/#{File.dirname(f)}"
369
370
  FileUtils.mkdir_p(file_dir)
371
+ num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
370
372
  FileUtils.mv(f, "#{dest_dir}/#{f}")
371
373
  end
374
+ return num_of_new_files
372
375
  end
373
376
 
374
377
  def get_experiment_events_log_via_kubectl(exp, namespace)
@@ -376,7 +379,7 @@ parameters:
376
379
  result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
377
380
 
378
381
  pod_name = result.split(" ")[0]
379
- return false if pod_name.blank?
382
+ return false, 0 if pod_name.blank?
380
383
  FileUtils.mkdir_p(dest_dir)
381
384
  working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
382
385
  working_dir.strip!
@@ -399,16 +402,18 @@ parameters:
399
402
  end
400
403
  end
401
404
 
405
+ num_of_new_files = 0
402
406
  all_files.each do |file|
403
407
  file_dir = "#{dest_dir}/#{File.dirname(file)}"
404
408
  FileUtils.mkdir_p(file_dir)
409
+ num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
405
410
  res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
406
411
  end
407
412
 
408
- return true
413
+ return true, num_of_new_files
409
414
  rescue => e
410
415
  Cnvrg::Logger.log_error(e)
411
- return false
416
+ return false, 0
412
417
  end
413
418
  end
414
419
 
@@ -421,7 +421,7 @@ module Cnvrg
421
421
 
422
422
  def generate_idx(deploy: false, files: [])
423
423
  if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
424
- old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
424
+ old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
425
425
  else
426
426
  old_idx = {:tree => {}, :commit => nil}
427
427
  end
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.10.5'
2
+ VERSION = '1.10.6'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.5
4
+ version: 1.10.6
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun