cnvrg 1.9.9.9.7 → 1.10.8

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cb866c9448de7ea2062759a1f9cd9bf0f9db23b3fae06c8b61d178dbd1a227c9
4
- data.tar.gz: dfc63cef384e0a8df9a367e03f544462638f5d0b43913dfe40b5dd59429ea2f8
3
+ metadata.gz: d9e8e76336bf9862fca95649740a7b5a5148565469ece725f4760041db28ffc4
4
+ data.tar.gz: f65852f97fb1294258c3c6b4b6941fb62248eeb1691d432e47bb4e2acb60a313
5
5
  SHA512:
6
- metadata.gz: 760d90fb1e7516a3e2f9ae72a7662ff02ae0e6324a883a17471ad5edacccc252f3b127a5bddf7ec0afcff6d42fe3b3f5596a6697fa058f5740ff0799894934b0
7
- data.tar.gz: ae6196d1a37f4bc68534aa728199d93500ee8299c99a896452158d22cf1e73578cf277ba0742f3e302ef9e677c466a335e9dac214c608e15c8ed088d870ddb60
6
+ metadata.gz: c5be8e6c872dd256560805d8146b3e2d7672a72844b932081dddc94c62e3fde4f9d7f66dd18d4f882c005c9e6e2183a91de689254b6dce07c265932058a4a1b0
7
+ data.tar.gz: 345eee06f18b24a30826665c7803b1275bdbc33d94779e5e37812cce7f95738bb819c5e2064871618145a27e72d133373e2493fb57f63b1c85ce8072c6057a54
@@ -212,7 +212,6 @@ module Cnvrg
212
212
  return nil
213
213
  rescue SignalException
214
214
  return false
215
-
216
215
  end
217
216
 
218
217
  end
@@ -2,13 +2,76 @@ module Cnvrg
2
2
  class API_V2 < API
3
3
  ENDPOINT_VERSION = 'v2'
4
4
 
5
+ class CnvrgAPIError < StandardError; end
6
+
5
7
  def self.endpoint_uri
6
8
  api = get_api()
7
9
  return "#{api}/#{Cnvrg::API_V2::ENDPOINT_VERSION}"
8
10
  end
9
11
 
12
+ def self.request(resource, method = 'GET', data = {}, parse_request = true)
13
+ resource = URI::encode resource
14
+ n = Netrc.read
15
+ if n['cnvrg.io'].nil?
16
+ puts 'You\'re not logged in'
17
+ puts 'Please log in via `cnvrg login`'
18
+ return
19
+ end
20
+
21
+ _, pass = n[Cnvrg::Helpers.netrc_domain]
22
+
23
+ conn = Faraday.new endpoint_uri, :ssl => {:verify => !!Helpers.is_verify_ssl}
24
+ conn.headers['Auth-Token'] = pass
25
+ conn.headers['User-Agent'] = Cnvrg::API::USER_AGENT
26
+ conn.headers['Content-Type'] = "application/json"
27
+ conn.options.timeout = 420
28
+ conn.options.open_timeout = 180
29
+
30
+ 20.times do
31
+ begin
32
+ response = send_request conn, resource, method, data
33
+ Cnvrg::API.parse_version response
34
+ is_response_success response
35
+ if parse_request
36
+ return JSON.parse(response.body)
37
+ else
38
+ return response.body
39
+ end
40
+ rescue CnvrgAPIError => e
41
+ raise e
42
+ rescue => e
43
+ Cnvrg::Logger.log_error e
44
+ sleep 0.5
45
+ retry
46
+ end
47
+ end
48
+ rescue CnvrgAPIError => e
49
+ Cnvrg::Logger.log_error e
50
+ raise e
51
+ rescue SignalException
52
+ return false
53
+ rescue => e
54
+ Cnvrg::Logger.log_error e
55
+ return nil
56
+ end
57
+
58
+ private
59
+
60
+ def self.send_request(conn, resource, method, data)
61
+ case method
62
+ when 'GET'
63
+ conn.get resource, data
64
+ when 'POST'
65
+ conn.post resource, data.to_json
66
+ when 'PUT'
67
+ conn.put resource, data.to_json
68
+ when 'DELETE'
69
+ conn.delete resource, data
70
+ end
71
+ end
72
+
10
73
  def self.is_response_success(response)
11
- raise Exception.new("Bad status in response #{response.status}") if response.status != 200
74
+ raise CnvrgAPIError.new(JSON(response.body)['errors']) if response.status != 200
12
75
  end
13
76
  end
14
- end
77
+ end
@@ -49,6 +49,7 @@ require 'cnvrg/job_cli'
49
49
  require 'cnvrg/job_ssh'
50
50
  require 'cnvrg/connect_job_ssh'
51
51
  require 'cnvrg/api_v2'
52
+ require 'rubygems/package'
52
53
 
53
54
  class Thor
54
55
  module Base
@@ -857,7 +858,7 @@ module Cnvrg
857
858
  method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
858
859
  method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
859
860
  method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
860
- def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false)
861
+ def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
861
862
  begin
862
863
  verify_logged_in(false)
863
864
  log_start(__method__, args, options)
@@ -913,7 +914,7 @@ module Cnvrg
913
914
 
914
915
  while files['keys'].length > 0
915
916
  Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
916
- @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten)
917
+ @files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
917
918
 
918
919
  downloaded_files += files['keys'].length
919
920
  files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
@@ -1267,7 +1268,7 @@ module Cnvrg
1267
1268
  @datafiles = Cnvrg::Datafiles.new(owner, slug, dataset: @dataset)
1268
1269
 
1269
1270
  # Init a new commit
1270
- response = @datafiles.start_commit(false, true, chunks: 1, message: message )
1271
+ response = @datafiles.start_commit(false, false, chunks: 1, message: message )
1271
1272
  unless response #means we failed in the start commit.
1272
1273
  raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
1273
1274
  end
@@ -1846,6 +1847,7 @@ module Cnvrg
1846
1847
  method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
1847
1848
  method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
1848
1849
  method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
1850
+ method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
1849
1851
  def clone(project_url)
1850
1852
  begin
1851
1853
  verify_logged_in(false)
@@ -1856,7 +1858,7 @@ module Cnvrg
1856
1858
  owner = url_parts[project_index - 1]
1857
1859
  remote = options["remote"] || false
1858
1860
  soft = options["soft"] || false
1859
-
1861
+ threads = options[:threads] || Cnvrg::Helpers.parallel_threads
1860
1862
 
1861
1863
  response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
1862
1864
  Cnvrg::CLI.is_response_success(response)
@@ -1907,9 +1909,10 @@ module Cnvrg
1907
1909
  idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
1908
1910
  log_message("Downloading files", Thor::Shell::Color::BLUE)
1909
1911
  progressbar = @files.create_progressbar(files.size, "Clone Progress")
1910
- @files.download_files(files, commit_sha1, progress: progressbar)
1912
+ @files.download_files(files, commit_sha1, progress: progressbar, threads: threads)
1911
1913
  progressbar.finish
1912
1914
  Project.verify_cnvrgignore_exist(project_name, remote)
1915
+ Cnvrg::Logger.log_info("Generating idx")
1913
1916
  @project.set_idx(idx)
1914
1917
  log_message("Done")
1915
1918
  log_message("Downloaded #{files.size} files")
@@ -2274,9 +2277,9 @@ module Cnvrg
2274
2277
  method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
2275
2278
  method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
2276
2279
  method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
2277
- method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
2280
+ method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
2278
2281
 
2279
- def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=100)
2282
+ def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=1000)
2280
2283
  begin
2281
2284
  # we are passing "force" twice.. doesnt really make sense :\\
2282
2285
  verify_logged_in(true)
@@ -2973,7 +2976,7 @@ module Cnvrg
2973
2976
  method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
2974
2977
  method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
2975
2978
  method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
2976
- method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 100
2979
+ method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
2977
2980
 
2978
2981
  def sync(direct = true)
2979
2982
  verify_logged_in(true) if direct
@@ -3043,6 +3046,7 @@ module Cnvrg
3043
3046
  method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
3044
3047
  method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
3045
3048
  method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
3049
+ method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
3046
3050
 
3047
3051
  def run(*cmd)
3048
3052
  verify_logged_in(true)
@@ -3077,6 +3081,7 @@ module Cnvrg
3077
3081
  email_notification_error = options["notify_on_error"]
3078
3082
  email_notification_success = options["notify_on_success"]
3079
3083
  emails = options["emails"]
3084
+ wait = options["wait"]
3080
3085
 
3081
3086
  if !data.present? and data_query.present?
3082
3087
  log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3122,14 +3127,14 @@ module Cnvrg
3122
3127
  invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
3123
3128
  :schedule => schedule, :recurring => recurring, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
3124
3129
  :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
3125
- :max_time => max_time,
3130
+ :max_time => max_time,
3126
3131
  :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
3127
3132
  :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
3128
3133
  :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
3129
- :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
3134
+ :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
3135
+
3130
3136
  return
3131
3137
  end
3132
-
3133
3138
  end
3134
3139
 
3135
3140
  desc '', '', :hide => true
@@ -3449,6 +3454,7 @@ module Cnvrg
3449
3454
  method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
3450
3455
  method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
3451
3456
  method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
3457
+ method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
3452
3458
 
3453
3459
  def exec_remote(*cmd)
3454
3460
 
@@ -3510,7 +3516,7 @@ module Cnvrg
3510
3516
  options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
3511
3517
  "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
3512
3518
  "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
3513
- "requirements", "prerun", "email_notification_error", "email_notification_success", "emails")
3519
+ "requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
3514
3520
  exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
3515
3521
  command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
3516
3522
  commit_to_run = options["commit"] || nil
@@ -3588,11 +3594,44 @@ module Cnvrg
3588
3594
  end
3589
3595
 
3590
3596
  log_message(str, Thor::Shell::Color::GREEN)
3597
+
3598
+ exit_status = 0
3599
+
3600
+ if options['wait']
3601
+ while true
3602
+ tries = 0
3603
+ begin
3604
+ result = Cnvrg::API_V2.request(
3605
+ "#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/exit_status",
3606
+ 'GET',
3607
+ { grid: res["result"]["grid"]}
3608
+ )
3609
+
3610
+ exit_statuses = result.values
3611
+ if exit_statuses.include? nil
3612
+ system("clear") || system("cls")
3613
+ msg = "#{Time.current}: waiting for experiment to finish"
3614
+ msg = "#{Time.current}: waiting for all experiments to finish" if res["result"]["grid"]
3615
+ puts msg
3616
+ sleep 3
3617
+ else
3618
+ result.each do |slug, exit_status|
3619
+ puts "Experiment #{slug} was exited with status #{exit_status}"
3620
+ end
3621
+ break
3622
+ end
3623
+ rescue => e
3624
+ log_error(e)
3625
+ log_message("Error occurred, retrying", Thor::Shell::Color::RED)
3626
+ sleep 3
3627
+ tries += 1
3628
+ retry if tries <= 5
3629
+ end
3630
+ end
3631
+ end
3591
3632
 
3592
- exit(0)
3593
- # end
3633
+ exit(exit_status.to_i)
3594
3634
  end
3595
-
3596
3635
  rescue => e
3597
3636
  log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
3598
3637
  log_error(e)
@@ -3608,6 +3647,7 @@ module Cnvrg
3608
3647
  exit(1)
3609
3648
  end
3610
3649
  end
3650
+
3611
3651
  desc 'deploy', 'Deploys model to production'
3612
3652
  method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
3613
3653
  method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
@@ -4816,17 +4856,19 @@ module Cnvrg
4816
4856
  exp_name = exp["title"]
4817
4857
  if exp["end_commit"].present? and exp["status"] != "Ongoing"
4818
4858
  log_message("#{exp_name} has ended, getting files from end commit", Thor::Shell::Color::BLUE)
4819
- Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
4859
+ num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project)
4820
4860
  exps_map[exp_slug] = exp
4821
4861
  else
4822
4862
  log_message("#{exp_name} is running should get logs", Thor::Shell::Color::BLUE)
4823
- success = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
4863
+ success, num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_via_kubectl(exp, namespace)
4824
4864
  if !success and exp["last_successful_commit"].present? and !copied_commits.include?(exp["last_successful_commit"])
4825
4865
  log_message("Failed to get kube files, using last commit", Thor::Shell::Color::BLUE)
4826
- Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
4866
+ num_of_new_files = Cnvrg::Helpers.get_experiment_events_log_from_server(exp, @project, commit: exp["last_successful_commit"])
4827
4867
  copied_commits << exp["last_successful_commit"]
4828
4868
  end
4829
4869
  end
4870
+
4871
+ log_message("New tf files copied", Thor::Shell::Color::BLUE) if num_of_new_files > 0
4830
4872
  rescue => e
4831
4873
  Cnvrg::Logger.log_error(e)
4832
4874
  end
@@ -154,6 +154,7 @@ module Cnvrg
154
154
  method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
155
155
  method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
156
156
  method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
157
+ method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
157
158
  def clone(dataset_url)
158
159
  cli = Cnvrg::CLI.new()
159
160
  only_tree =options[:only_tree]
@@ -163,6 +164,7 @@ module Cnvrg
163
164
  remote = options[:remote]
164
165
  soft = options[:soft]
165
166
  flatten = options[:flatten]
167
+ threads = options[:threads]
166
168
  cli.clone_data(
167
169
  dataset_url,
168
170
  only_tree=only_tree,
@@ -172,7 +174,8 @@ module Cnvrg
172
174
  remote=remote,
173
175
  flatten: flatten,
174
176
  relative: options[:relative],
175
- soft: soft
177
+ soft: soft,
178
+ threads: threads
176
179
  )
177
180
  end
178
181
 
@@ -409,8 +409,6 @@ module Cnvrg
409
409
  tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
410
410
  results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
411
411
 
412
- # puts "Got #{results['files'].size} files to upload from #{files_chunk.size} files"
413
-
414
412
  if results['files'].blank?
415
413
  progress_mutex.synchronize { progressbar.progress += tree.keys.length }
416
414
  next
@@ -1284,11 +1282,11 @@ module Cnvrg
1284
1282
  end
1285
1283
  end
1286
1284
 
1287
- def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false)
1285
+ def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
1288
1286
  begin
1289
1287
  refresh_storage_token
1290
1288
  parallel_options = {
1291
- in_threads: ParallelThreads,
1289
+ in_threads: threads,
1292
1290
  isolation: true
1293
1291
  }
1294
1292
  Parallel.map(files["keys"], parallel_options) do |f|
@@ -544,8 +544,12 @@ module Cnvrg
544
544
  def generate_chunked_idx(list_files = [], threads: 15, prefix: '')
545
545
  tree = {}
546
546
  Parallel.map(list_files, in_threads: threads) do |file|
547
- #check if prefix exists do prefix/path otherwise path
548
- label = file.gsub(self.local_path + "/", "")
547
+
548
+ # Fix for root path issue
549
+ safe_path = file
550
+ safe_path = file[1..-1] if file.start_with? "/"
551
+
552
+ label = safe_path.gsub(self.local_path + "/", "")
549
553
  label = "#{prefix}/#{label}" if prefix.present?
550
554
  if not Cnvrg::Files.valid_file_name?(label)
551
555
  raise StandardError.new("#{label} is not a valid file name.")
@@ -558,9 +562,16 @@ module Cnvrg
558
562
  file_size = File.size(file).to_f
559
563
  mime_type = MimeMagic.by_path(file)
560
564
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
561
- relative_path = file.gsub(/^#{@local_path + "/"}/, "")
565
+ relative_path = safe_path.gsub(/^#{@local_path + "/"}/, "")
562
566
  relative_path = "#{prefix}/#{relative_path}" if prefix.present?
563
- tree[label] = {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path: file, relative_path: relative_path}
567
+ tree[label] = {
568
+ sha1: sha1,
569
+ file_name: file_name,
570
+ file_size: file_size,
571
+ content_type: content_type,
572
+ absolute_path: file,
573
+ relative_path: relative_path
574
+ }
564
575
  end
565
576
  end
566
577
  if prefix.present? #add the prefix as dirs to the files
@@ -723,7 +723,7 @@ module Cnvrg
723
723
  @progressbar
724
724
  end
725
725
 
726
- def download_files(files, commit, postfix: '', progress: nil)
726
+ def download_files(files, commit, postfix: '', progress: nil, threads: 15)
727
727
  return if files.blank?
728
728
  if Cnvrg::Helpers.server_version < 1
729
729
  Cnvrg::Logger.log_info("Download files from older server.")
@@ -733,7 +733,7 @@ module Cnvrg
733
733
  unless Cnvrg::CLI.is_response_success(res, false)
734
734
  raise SignalException.new("Cant download files from the server.")
735
735
  end
736
- self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
736
+ self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress, threads: threads)
737
737
  end
738
738
 
739
739
 
@@ -750,7 +750,7 @@ module Cnvrg
750
750
  conflicted.each{|file| self.delete_conflict(file); progress.progress += 1 if progress.present?}
751
751
  end
752
752
 
753
- def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
753
+ def download_multpile_files_s3(files, project_home, postfix: '', progress: nil, threads: 15)
754
754
  begin
755
755
  props = {}
756
756
  client = props[:client]
@@ -759,7 +759,7 @@ module Cnvrg
759
759
  bucket = props[:bucket]
760
760
  download_succ_count = 0
761
761
  parallel_options = {
762
- in_threads: Cnvrg::Helpers.parallel_threads,
762
+ in_threads: threads,
763
763
  isolation: true
764
764
  }
765
765
 
@@ -2,15 +2,8 @@ module Cnvrg
2
2
  module Helpers
3
3
 
4
4
  extend self
5
- def parallel_threads()
6
- threads = ENV["CNVRG_PARALLEL_THREADS"].to_i
7
-
8
-
9
- ### if empty, default will be 15 threads
10
- threads = threads > 0 ? threads : 15
11
-
12
- ### set max threads to be 100k
13
- [threads, 100000].min
5
+ def parallel_threads
6
+ 15
14
7
  end
15
8
 
16
9
  def self.parallel_options
@@ -371,11 +364,14 @@ parameters:
371
364
  @files.download_files(files, commit_sha1, progress: nil)
372
365
  FileUtils.rm_rf("#{dest_dir}")
373
366
  FileUtils.mkdir_p(dest_dir)
367
+ num_of_new_files = 0
374
368
  files.each do |f|
375
369
  file_dir = "#{dest_dir}/#{File.dirname(f)}"
376
370
  FileUtils.mkdir_p(file_dir)
371
+ num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{f}")
377
372
  FileUtils.mv(f, "#{dest_dir}/#{f}")
378
373
  end
374
+ return num_of_new_files
379
375
  end
380
376
 
381
377
  def get_experiment_events_log_via_kubectl(exp, namespace)
@@ -383,7 +379,7 @@ parameters:
383
379
  result = `kubectl -n #{namespace} get pods | grep #{exp["slug"]}`
384
380
 
385
381
  pod_name = result.split(" ")[0]
386
- return false if pod_name.blank?
382
+ return false, 0 if pod_name.blank?
387
383
  FileUtils.mkdir_p(dest_dir)
388
384
  working_dir = `kubectl -n #{namespace} exec #{pod_name} -c agent -- pwd`
389
385
  working_dir.strip!
@@ -406,16 +402,18 @@ parameters:
406
402
  end
407
403
  end
408
404
 
405
+ num_of_new_files = 0
409
406
  all_files.each do |file|
410
407
  file_dir = "#{dest_dir}/#{File.dirname(file)}"
411
408
  FileUtils.mkdir_p(file_dir)
409
+ num_of_new_files += 1 unless File.exist?("#{dest_dir}/#{file}")
412
410
  res = `kubectl -n #{namespace} cp #{pod_name}:#{file} -c agent #{dest_dir}/#{file}`
413
411
  end
414
412
 
415
- return true
413
+ return true, num_of_new_files
416
414
  rescue => e
417
415
  Cnvrg::Logger.log_error(e)
418
- return false
416
+ return false, 0
419
417
  end
420
418
  end
421
419
 
@@ -421,7 +421,7 @@ module Cnvrg
421
421
 
422
422
  def generate_idx(deploy: false, files: [])
423
423
  if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
424
- old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
424
+ old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml") rescue {:tree => {}, :commit => nil}
425
425
  else
426
426
  old_idx = {:tree => {}, :commit => nil}
427
427
  end
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.9.9.9.7'
2
+ VERSION = '1.10.8'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.9.9.9.7
4
+ version: 1.10.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-07-21 00:00:00.000000000 Z
13
+ date: 2020-08-20 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler