cnvrg 1.11.10 → 1.11.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/api.rb +22 -3
- data/lib/cnvrg/datafiles.rb +9 -1
- data/lib/cnvrg/downloader/client.rb +3 -1
- data/lib/cnvrg/helpers/agent.rb +7 -2
- data/lib/cnvrg/helpers/executer.rb +24 -10
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cc3f80382d5e1ebdddea819a74e92cd00806a273dfd4d58c484e2a91a7f86932
|
4
|
+
data.tar.gz: cfb22dc8d1548b7c6a1f0af0e588e8c03a62bcad67b06fc8c1573116df7db31b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3096470b6f4aa425b2804e12b1db724c33700d209387475fbee16977c89ac75d4794f3eea304de454936fca93b5a24df8ba57391f05490fc076abc2c2f3b2a20
|
7
|
+
data.tar.gz: dd3dfd71604961c548d197c72cda3a6dd586103e19d38dabc05cc1af2e8262d84bab4126dfe93fe46f920c8fb345508c6626c42b212a7ad5a02b6cca0893c0d2
|
data/lib/cnvrg/api.rb
CHANGED
@@ -65,6 +65,16 @@ module Cnvrg
|
|
65
65
|
response = conn.get "#{resource}", data
|
66
66
|
success = true
|
67
67
|
Cnvrg::API.parse_version(response)
|
68
|
+
if response.to_hash[:status].to_i != 200
|
69
|
+
Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
|
70
|
+
end
|
71
|
+
if [503, 502, 429].include?(response.to_hash[:status].to_i)
|
72
|
+
Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
|
73
|
+
success = false
|
74
|
+
sleep(5 * retries)
|
75
|
+
retries +=1
|
76
|
+
next
|
77
|
+
end
|
68
78
|
rescue => e
|
69
79
|
Cnvrg::Logger.log_error(e)
|
70
80
|
sleep(5)
|
@@ -95,11 +105,20 @@ module Cnvrg
|
|
95
105
|
response = conn.put "#{resource}", data.to_json if method.eql? 'PUT'
|
96
106
|
success = true
|
97
107
|
Cnvrg::API.parse_version(response)
|
98
|
-
|
108
|
+
if response.to_hash[:status].to_i != 200
|
109
|
+
Cnvrg::Logger.log_info("Got back bad status #{response.to_hash[:status]}")
|
110
|
+
end
|
111
|
+
if [503, 502, 429].include?(response.to_hash[:status].to_i)
|
112
|
+
Cnvrg::Logger.log_info("Got back status #{response.to_hash[:status]}, will retry in #{5 * retries} seconds")
|
113
|
+
success = false
|
114
|
+
sleep(5 * retries)
|
115
|
+
retries +=1
|
116
|
+
next
|
117
|
+
end
|
99
118
|
rescue => e
|
100
119
|
Cnvrg::Logger.log_error(e)
|
101
|
-
|
102
|
-
|
120
|
+
sleep(5)
|
121
|
+
retries +=1
|
103
122
|
end
|
104
123
|
end
|
105
124
|
if !success
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -11,6 +11,7 @@ module Cnvrg
|
|
11
11
|
|
12
12
|
LARGE_FILE=1024*1024*5
|
13
13
|
MULTIPART_SPLIT=10000000
|
14
|
+
RETRIES = ENV['UPLOAD_FILE_RETRIES'].try(:to_i) || 10
|
14
15
|
|
15
16
|
attr_reader :base_resource
|
16
17
|
|
@@ -256,6 +257,7 @@ module Cnvrg
|
|
256
257
|
end
|
257
258
|
|
258
259
|
def delete_file_chunk(commit_sha1, regex_list, chunk_size, offset)
|
260
|
+
retry_count = 0
|
259
261
|
begin
|
260
262
|
resp = Cnvrg::API.request(
|
261
263
|
@base_resource + "delete_files_by_chunk",
|
@@ -268,13 +270,19 @@ module Cnvrg
|
|
268
270
|
}
|
269
271
|
)
|
270
272
|
unless Cnvrg::CLI.is_response_success(resp, false)
|
271
|
-
Cnvrg::Logger.log_method(bind: binding)
|
272
273
|
raise Exception.new("Got an error message from server, #{resp.try(:fetch, "message")}")
|
273
274
|
end
|
274
275
|
return resp["total_changes"]
|
275
276
|
rescue => e
|
276
277
|
Cnvrg::Logger.log_method(bind: binding)
|
277
278
|
Cnvrg::Logger.log_error(e)
|
279
|
+
|
280
|
+
if retry_count < RETRIES
|
281
|
+
sleep(2**retry_count) # Exponential backoff
|
282
|
+
retry_count += 1
|
283
|
+
retry
|
284
|
+
end
|
285
|
+
|
278
286
|
raise e
|
279
287
|
end
|
280
288
|
end
|
@@ -14,11 +14,13 @@ module Cnvrg
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def extract_key_iv(sts_path)
|
17
|
-
count =
|
17
|
+
count = 0
|
18
18
|
begin
|
19
19
|
count += 1
|
20
20
|
sts = open(sts_path, {ssl_verify_mode: 0}).read rescue nil
|
21
21
|
rescue => e
|
22
|
+
backoff_time_seconds = backoff_time(count)
|
23
|
+
sleep backoff_time_seconds
|
22
24
|
Cnvrg::Logger.log_error(e)
|
23
25
|
retry if count <= 20
|
24
26
|
raise StandardError.new("Cant access storage: #{e.message}")
|
data/lib/cnvrg/helpers/agent.rb
CHANGED
@@ -75,9 +75,11 @@ class Cnvrg::Helpers::Agent
|
|
75
75
|
|
76
76
|
def exec!
|
77
77
|
log_internal("Command: #{@command} with slug: #{@slug} started!")
|
78
|
-
if
|
78
|
+
if @command.blank?
|
79
|
+
@exit_status = 0
|
80
|
+
elsif should_run?
|
79
81
|
send_logs(status: Status::STARTED)
|
80
|
-
periodic_thread
|
82
|
+
periodic_thread_handle = periodic_thread
|
81
83
|
execute_command
|
82
84
|
else
|
83
85
|
@exit_status = 127
|
@@ -86,6 +88,9 @@ class Cnvrg::Helpers::Agent
|
|
86
88
|
finish_log += " after #{@real_execution_retries} retries" if @real_execution_retries > 0
|
87
89
|
log_internal(finish_log)
|
88
90
|
send_logs(exit_status: @exit_status, status: Status::FINISHED)
|
91
|
+
if periodic_thread_handle.present?
|
92
|
+
periodic_thread_handle.join
|
93
|
+
end
|
89
94
|
end
|
90
95
|
|
91
96
|
def get_logs_to_send
|
@@ -120,17 +120,31 @@ class Cnvrg::Helpers::Executer
|
|
120
120
|
end
|
121
121
|
|
122
122
|
def init
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
123
|
+
retries = 0
|
124
|
+
success = false
|
125
|
+
puts("Agent started, connecting to #{Cnvrg::API.get_api}")
|
126
|
+
STDOUT.flush
|
127
|
+
while !success and retries < 100
|
128
|
+
begin
|
129
|
+
resp = Cnvrg::API.request(activity_url, "PUT", {stats: executer_stats})
|
130
|
+
machine_activity = resp["machine_activity"]
|
131
|
+
success = true
|
132
|
+
puts("Connected to server")
|
133
|
+
STDOUT.flush
|
134
|
+
Cnvrg::Logger.log_info("Got back machine activity #{machine_activity}")
|
135
|
+
if machine_activity.present? and @machine_activity != machine_activity
|
136
|
+
Cnvrg::Logger.log_info("Changing to machine activity #{machine_activity}")
|
137
|
+
machine_activity_yml = {slug: machine_activity}
|
138
|
+
File.open("/conf/.machine_activity.yml", "w+") {|f| f.write machine_activity_yml.to_yaml}
|
139
|
+
@machine_activity = machine_activity
|
140
|
+
end
|
141
|
+
rescue => e
|
142
|
+
Cnvrg::Logger.log_error(e)
|
143
|
+
Cnvrg::Logger.info("Sleeping for #{5 * retries}")
|
144
|
+
sleep(5 * retries)
|
145
|
+
retries +=1
|
146
|
+
end
|
131
147
|
end
|
132
|
-
rescue => e
|
133
|
-
Cnvrg::Logger.log_error(e)
|
134
148
|
end
|
135
149
|
|
136
150
|
def polling_thread
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.11.
|
4
|
+
version: 1.11.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-01-06 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|