cnvrg 1.11.8 → 1.11.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/api.rb +22 -3
- data/lib/cnvrg/cli.rb +12 -20
- data/lib/cnvrg/datafiles.rb +9 -1
- data/lib/cnvrg/downloader/client.rb +3 -1
- data/lib/cnvrg/experiment.rb +2 -1
- data/lib/cnvrg/files.rb +15 -15
- data/lib/cnvrg/helpers/agent.rb +7 -2
- data/lib/cnvrg/helpers/executer.rb +24 -10
- data/lib/cnvrg/version.rb +1 -1
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 4efa915d1f57cfba945987168eb4cfd118bfee50e8d5e3fa76dd08592176e054
|
4
|
+
data.tar.gz: 8f2b8cd5916f6281c25791f36791c6cc3052f0b4fe7a1094661c635f63eb1b9d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 24459de6ce2e782816f68f891e79f8b4fe75d98571983321358619be51fd718055581aea9c4db8a850aea3af97ada9b50207b8278490088c42dce60407b1dc8e
|
7
|
+
data.tar.gz: 03a9f04384ddfebc7e3ee9b0eb10361d808e686286b77ba26753969284e7ae707c147aab7221325d89f9bd22151d378f2865a96c1b01d2d45657f488d2ee97ee
|
data/lib/cnvrg/api.rb
CHANGED
@@ -65,6 +65,16 @@ module Cnvrg
|
|
65
65
|
response = conn.get "#{resource}", data
|
66
66
|
success = true
|
67
67
|
Cnvrg::API.parse_version(response)
|
68
|
+
if response.to_hash[:status].to_i != 200
|
69
|
+
Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
|
70
|
+
end
|
71
|
+
if [503, 502, 429].include?(response.to_hash[:status].to_i)
|
72
|
+
Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
|
73
|
+
success = false
|
74
|
+
sleep(5 * retries)
|
75
|
+
retries +=1
|
76
|
+
next
|
77
|
+
end
|
68
78
|
rescue => e
|
69
79
|
Cnvrg::Logger.log_error(e)
|
70
80
|
sleep(5)
|
@@ -95,11 +105,20 @@ module Cnvrg
|
|
95
105
|
response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
|
96
106
|
success = true
|
97
107
|
Cnvrg::API.parse_version(response)
|
98
|
-
|
108
|
+
if response.to_hash[:status].to_i != 200
|
109
|
+
Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
|
110
|
+
end
|
111
|
+
if [503, 502, 429].include?(response.to_hash[:status].to_i)
|
112
|
+
Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
|
113
|
+
success = false
|
114
|
+
sleep(5 * retries)
|
115
|
+
retries +=1
|
116
|
+
next
|
117
|
+
end
|
99
118
|
rescue => e
|
100
119
|
Cnvrg::Logger.log_error(e)
|
101
|
-
|
102
|
-
|
120
|
+
sleep(5)
|
121
|
+
retries +=1
|
103
122
|
end
|
104
123
|
end
|
105
124
|
if !success
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -3068,6 +3068,7 @@ module Cnvrg
|
|
3068
3068
|
method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
|
3069
3069
|
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
|
3070
3070
|
method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
|
3071
|
+
method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
|
3071
3072
|
|
3072
3073
|
def run(*cmd)
|
3073
3074
|
verify_logged_in(true)
|
@@ -3076,6 +3077,7 @@ module Cnvrg
|
|
3076
3077
|
sync_before = options["sync_before"]
|
3077
3078
|
sync_after = options["sync_after"]
|
3078
3079
|
log = options["log"]
|
3080
|
+
debug = options["debug"]
|
3079
3081
|
title = options["title"]
|
3080
3082
|
commit = options["commit"] || nil
|
3081
3083
|
email_notification = options["email_notification"]
|
@@ -3109,6 +3111,7 @@ module Cnvrg
|
|
3109
3111
|
wait = false
|
3110
3112
|
end
|
3111
3113
|
|
3114
|
+
|
3112
3115
|
if !data.present? and data_query.present?
|
3113
3116
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
3114
3117
|
exit(1)
|
@@ -3155,7 +3158,7 @@ module Cnvrg
|
|
3155
3158
|
:image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
|
3156
3159
|
:max_time => max_time,
|
3157
3160
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
3158
|
-
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
3161
|
+
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch, :debug => debug,
|
3159
3162
|
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
|
3160
3163
|
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
|
3161
3164
|
|
@@ -3379,10 +3382,10 @@ module Cnvrg
|
|
3379
3382
|
if @project.is_git
|
3380
3383
|
output_dir = output_dir || @exp.output_dir
|
3381
3384
|
if output_dir.present?
|
3382
|
-
upload(false, false, true, ignore, true,
|
3385
|
+
upload(false, false, true, ignore, true, false, output_dir, "Experiment", @exp.slug, true )
|
3383
3386
|
end
|
3384
3387
|
else
|
3385
|
-
upload(false, false, true, ignore, true,
|
3388
|
+
upload(false, false, true, ignore, true, false, nil, "Experiment", @exp.slug, true )
|
3386
3389
|
end
|
3387
3390
|
end
|
3388
3391
|
|
@@ -3481,6 +3484,7 @@ module Cnvrg
|
|
3481
3484
|
method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
|
3482
3485
|
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
|
3483
3486
|
method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
|
3487
|
+
method_option :debug, :type => :boolean, :aliases => ["--debug"], :default => true
|
3484
3488
|
|
3485
3489
|
def exec_remote(*cmd)
|
3486
3490
|
|
@@ -3498,6 +3502,7 @@ module Cnvrg
|
|
3498
3502
|
data_query = options["data_query"] || nil
|
3499
3503
|
sync_before = options["sync_before"]
|
3500
3504
|
force = options["force"]
|
3505
|
+
debug = options["debug"]
|
3501
3506
|
prerun = options["prerun"]
|
3502
3507
|
requirements = options["requirements"]
|
3503
3508
|
email_notification_error = options["email_notification_error"]
|
@@ -3541,8 +3546,8 @@ module Cnvrg
|
|
3541
3546
|
local_folders_options = options["local_folders"]
|
3542
3547
|
options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
3543
3548
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
3544
|
-
"data_query", "git_commit","git_branch",
|
3545
|
-
"requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
|
3549
|
+
"data_query", "git_commit","git_branch","restart_if_stuck","local_folders","output_dir", "commit", "datasets",
|
3550
|
+
"requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait","debug")
|
3546
3551
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
3547
3552
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
3548
3553
|
commit_to_run = options["commit"] || nil
|
@@ -3585,20 +3590,6 @@ module Cnvrg
|
|
3585
3590
|
end
|
3586
3591
|
end
|
3587
3592
|
|
3588
|
-
if command.include? "'"
|
3589
|
-
oc = command.to_enum(:scan, /'/).map {Regexp.last_match}
|
3590
|
-
pairs = oc.enum_for(:each_slice, 2).to_a
|
3591
|
-
pairs.each_with_index do |p, i|
|
3592
|
-
add = 0
|
3593
|
-
if i != 0
|
3594
|
-
add = 2 * i
|
3595
|
-
end
|
3596
|
-
total_loc = command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add]
|
3597
|
-
command[p[0].offset(0)[0] + add..p[1].offset(0)[0] + add] = "\"#{total_loc}\""
|
3598
|
-
end
|
3599
|
-
|
3600
|
-
|
3601
|
-
end
|
3602
3593
|
log_message("Running remote experiment", Thor::Shell::Color::BLUE)
|
3603
3594
|
exp = Experiment.new(project.owner, project.slug)
|
3604
3595
|
if forced_commit and (commit_to_run.nil? or commit_to_run.empty?)
|
@@ -3609,7 +3600,7 @@ module Cnvrg
|
|
3609
3600
|
|
3610
3601
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
3611
3602
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
3612
|
-
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
|
3603
|
+
data_query, git_commit, git_branch,debug, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
|
3613
3604
|
email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
|
3614
3605
|
if Cnvrg::CLI.is_response_success(res)
|
3615
3606
|
check = Helpers.checkmark()
|
@@ -3661,6 +3652,7 @@ module Cnvrg
|
|
3661
3652
|
sleep 3
|
3662
3653
|
tries += 1
|
3663
3654
|
retry if tries <= 5
|
3655
|
+
exit(1)
|
3664
3656
|
end
|
3665
3657
|
end
|
3666
3658
|
end
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -11,6 +11,7 @@ module Cnvrg
|
|
11
11
|
|
12
12
|
LARGE_FILE=1024*1024*5
|
13
13
|
MULTIPART_SPLIT=10000000
|
14
|
+
RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 10
|
14
15
|
|
15
16
|
attr_reader :base_resource
|
16
17
|
|
@@ -256,6 +257,7 @@ module Cnvrg
|
|
256
257
|
end
|
257
258
|
|
258
259
|
def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
|
260
|
+
retry_count = 0
|
259
261
|
begin
|
260
262
|
resp = Cnvrg::API.request(
|
261
263
|
@base_resource + "delete_files_by_chunk",
|
@@ -268,13 +270,19 @@ module Cnvrg
|
|
268
270
|
}
|
269
271
|
)
|
270
272
|
unless Cnvrg::CLI.is_response_success(resp, false)
|
271
|
-
Cnvrg::Logger.log_method(bind: binding)
|
272
273
|
raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
|
273
274
|
end
|
274
275
|
return resp["total_changes"]
|
275
276
|
rescue => e
|
276
277
|
Cnvrg::Logger.log_method(bind: binding)
|
277
278
|
Cnvrg::Logger.log_error(e)
|
279
|
+
|
280
|
+
if retry_count < RETRIES
|
281
|
+
sleep(2**retry_count) # Exponential backoff
|
282
|
+
retry_count += 1
|
283
|
+
retry
|
284
|
+
end
|
285
|
+
|
278
286
|
raise e
|
279
287
|
end
|
280
288
|
end
|
@@ -14,11 +14,13 @@ module Cnvrg
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def extract_key_iv(sts_path)
|
17
|
-
count =
|
17
|
+
count = 0
|
18
18
|
begin
|
19
19
|
count += 1
|
20
20
|
sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
|
21
21
|
rescue => e
|
22
|
+
backoff_time_seconds = backoff_time(count)
|
23
|
+
sleep backoff_time_seconds
|
22
24
|
Cnvrg::Logger.log_error(e)
|
23
25
|
retry if count <= 20
|
24
26
|
raise StandardError.new("Cant access storage: #{e.message}")
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -110,7 +110,7 @@ module Cnvrg
|
|
110
110
|
|
111
111
|
def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
|
112
112
|
sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
|
113
|
-
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
|
113
|
+
git_commit=nil, git_branch=nil,debug=true, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
|
114
114
|
email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
|
115
115
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
116
116
|
commit_sha1: commit_to_run,
|
@@ -119,6 +119,7 @@ module Cnvrg
|
|
119
119
|
local_timestamp:local_timestamp,
|
120
120
|
datasets: datasets,
|
121
121
|
grid: grid,
|
122
|
+
debug:debug,
|
122
123
|
path_to_cmd:path_to_cmd,dataset_slug:data,
|
123
124
|
dataset_commit: data_commit,max_time:max_time,
|
124
125
|
periodic_sync:periodic_sync, sync_before_terminate:sync_before_terminate,
|
data/lib/cnvrg/files.rb
CHANGED
@@ -751,6 +751,7 @@ module Cnvrg
|
|
751
751
|
end
|
752
752
|
|
753
753
|
def download_multpile_files_s3(files, project_home, postfix: '', progress: nil, threads: 15)
|
754
|
+
cli = Cnvrg::CLI.new()
|
754
755
|
begin
|
755
756
|
props = {}
|
756
757
|
client = props[:client]
|
@@ -774,11 +775,15 @@ module Cnvrg
|
|
774
775
|
file_path = f["name"]
|
775
776
|
if file_path.end_with? "/"
|
776
777
|
# dir
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
778
|
+
begin
|
779
|
+
if download_dir(file_path, file_path, project_home)
|
780
|
+
download_succ_count += 1
|
781
|
+
else
|
782
|
+
raise SignalException.new("Could not create directory #{file_path}.")
|
783
|
+
end
|
784
|
+
rescue => e
|
785
|
+
cli.log_info("Could not create directory #{file_path}. error: #{e.message}", Thor::Shell::Color::RED)
|
786
|
+
raise e
|
782
787
|
end
|
783
788
|
else
|
784
789
|
file_path += postfix
|
@@ -793,25 +798,20 @@ module Cnvrg
|
|
793
798
|
progress.progress += 1 if progress.present?
|
794
799
|
download_succ_count += 1
|
795
800
|
rescue => e
|
796
|
-
|
797
|
-
raise
|
801
|
+
cli.log_info("Could not download file #{file_path}. error: #{e.message}", Thor::Shell::Color::RED)
|
802
|
+
raise e
|
798
803
|
end
|
799
|
-
|
800
|
-
|
801
|
-
|
802
804
|
end
|
803
805
|
end
|
804
806
|
if download_succ_count == files["keys"].size
|
805
807
|
return Cnvrg::Result.new(true,"Done.\nDownloaded #{download_succ_count} files")
|
806
808
|
end
|
807
809
|
rescue => e
|
808
|
-
|
810
|
+
cli.log_error(e)
|
811
|
+
raise e
|
809
812
|
end
|
810
|
-
|
811
|
-
|
812
|
-
|
813
|
-
|
814
813
|
end
|
814
|
+
|
815
815
|
def download_file(absolute_path, relative_path, project_home, conflict=false)
|
816
816
|
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
817
817
|
Cnvrg::CLI.is_response_success(res, false)
|
data/lib/cnvrg/helpers/agent.rb
CHANGED
@@ -75,9 +75,11 @@ class Cnvrg::Helpers::Agent
|
|
75
75
|
|
76
76
|
def exec!
|
77
77
|
log_internal("Command: #{@command} with slug: #{@slug} started!")
|
78
|
-
if
|
78
|
+
if @command.blank?
|
79
|
+
@exit_status = 0
|
80
|
+
elsif should_run?
|
79
81
|
send_logs(status: Status::STARTED)
|
80
|
-
periodic_thread
|
82
|
+
periodic_thread_handle = periodic_thread
|
81
83
|
execute_command
|
82
84
|
else
|
83
85
|
@exit_status = 127
|
@@ -86,6 +88,9 @@ class Cnvrg::Helpers::Agent
|
|
86
88
|
finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
|
87
89
|
log_internal(finish_log)
|
88
90
|
send_logs(exit_status: @exit_status, status: Status::FINISHED)
|
91
|
+
if periodic_thread_handle.present?
|
92
|
+
periodic_thread_handle.join
|
93
|
+
end
|
89
94
|
end
|
90
95
|
|
91
96
|
def get_logs_to_send
|
@@ -120,17 +120,31 @@ class Cnvrg::Helpers::Executer
|
|
120
120
|
end
|
121
121
|
|
122
122
|
def init
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
123
|
+
retries = 0
|
124
|
+
success = false
|
125
|
+
puts("Agent started, connecting to #{Cnvrg::API.get_api}")
|
126
|
+
STDOUT.flush
|
127
|
+
while !success and retries < 100
|
128
|
+
begin
|
129
|
+
resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
|
130
|
+
machine_activity = resp["machine_activity"]
|
131
|
+
success = true
|
132
|
+
puts("Connected to server")
|
133
|
+
STDOUT.flush
|
134
|
+
Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
|
135
|
+
if machine_activity.present? and @machine_activity != machine_activity
|
136
|
+
Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
|
137
|
+
machine_activity_yml = {slug: machine_activity}
|
138
|
+
File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
|
139
|
+
@machine_activity = machine_activity
|
140
|
+
end
|
141
|
+
rescue => e
|
142
|
+
Cnvrg::Logger.log_error(e)
|
143
|
+
Cnvrg::Logger.info("Sleeping for #{5 * retries}")
|
144
|
+
sleep(5 * retries)
|
145
|
+
retries +=1
|
146
|
+
end
|
131
147
|
end
|
132
|
-
rescue => e
|
133
|
-
Cnvrg::Logger.log_error(e)
|
134
148
|
end
|
135
149
|
|
136
150
|
def polling_thread
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.11.
|
4
|
+
version: 1.11.14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-01-22 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -453,7 +453,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
453
453
|
- !ruby/object:Gem::Version
|
454
454
|
version: '0'
|
455
455
|
requirements: []
|
456
|
-
rubygems_version: 3.
|
456
|
+
rubygems_version: 3.1.2
|
457
457
|
signing_key:
|
458
458
|
specification_version: 4
|
459
459
|
summary: A CLI tool for interacting with cnvrg.io.
|