cnvrg 1.9.9.9 → 1.10.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/api.rb +0 -1
- data/lib/cnvrg/api_v2.rb +65 -2
- data/lib/cnvrg/cli.rb +53 -16
- data/lib/cnvrg/data.rb +4 -1
- data/lib/cnvrg/datafiles.rb +2 -2
- data/lib/cnvrg/files.rb +52 -46
- data/lib/cnvrg/helpers.rb +2 -9
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2c24bbf4a254e135ea5de9795e5394c0b480ddfdce161faeb60db440c398f129
|
4
|
+
data.tar.gz: 292d5f3fac1c85ed16d2fbb254c8d8296246f0672f885cfb8723cda155e92272
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 44ffaa361515845b54a148d727385c224b29e539870abe2c7da88ab9357a9a69ae021ed1d3201bba92875895707c43aaa11c225918ac331c59b31086bf97b13e
|
7
|
+
data.tar.gz: fa2cec0111bf41e16efa30ae14e9c343fa2a29c8678660a2b7246ea3a1e3a1bb9a58ccf21955da849fc0362d9d1c756df7b8975ec497fdab5918f973d302aeeb
|
data/lib/cnvrg/api.rb
CHANGED
data/lib/cnvrg/api_v2.rb
CHANGED
@@ -2,13 +2,76 @@ module Cnvrg
|
|
2
2
|
class API_V2 < API
|
3
3
|
ENDPOINT_VERSION = 'v2'
|
4
4
|
|
5
|
+
class CnvrgAPIError < StandardError; end
|
6
|
+
|
5
7
|
def self.endpoint_uri
|
6
8
|
api = get_api()
|
7
9
|
return "#{api}/#{Cnvrg::API_V2::ENDPOINT_VERSION}"
|
8
10
|
end
|
9
11
|
|
12
|
+
def self.request(resource, method = 'GET', data = {}, parse_request = true)
|
13
|
+
resource = URI::encode resource
|
14
|
+
n = Netrc.read
|
15
|
+
if n['cnvrg.io'].nil?
|
16
|
+
puts 'You\'re not logged in'
|
17
|
+
puts 'Please log in via `cnvrg login`'
|
18
|
+
return
|
19
|
+
end
|
20
|
+
|
21
|
+
_, pass = n[Cnvrg::Helpers.netrc_domain]
|
22
|
+
|
23
|
+
conn = Faraday.new endpoint_uri, :ssl => {:verify => !!Helpers.is_verify_ssl}
|
24
|
+
conn.headers['Auth-Token'] = pass
|
25
|
+
conn.headers['User-Agent'] = Cnvrg::API::USER_AGENT
|
26
|
+
conn.headers['Content-Type'] = "application/json"
|
27
|
+
conn.options.timeout = 420
|
28
|
+
conn.options.open_timeout = 180
|
29
|
+
|
30
|
+
20.times do
|
31
|
+
begin
|
32
|
+
response = send_request conn, resource, method, data
|
33
|
+
Cnvrg::API.parse_version response
|
34
|
+
is_response_success response
|
35
|
+
if parse_request
|
36
|
+
return JSON.parse(response.body)
|
37
|
+
else
|
38
|
+
return response.body
|
39
|
+
end
|
40
|
+
rescue CnvrgAPIError => e
|
41
|
+
raise e
|
42
|
+
rescue => e
|
43
|
+
Cnvrg::Logger.log_error e
|
44
|
+
sleep 0.5
|
45
|
+
retry
|
46
|
+
end
|
47
|
+
end
|
48
|
+
rescue CnvrgAPIError => e
|
49
|
+
Cnvrg::Logger.log_error e
|
50
|
+
raise e
|
51
|
+
rescue SignalException
|
52
|
+
return false
|
53
|
+
rescue => e
|
54
|
+
Cnvrg::Logger.log_error e
|
55
|
+
return nil
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
def self.send_request(conn, resource, method, data)
|
61
|
+
case method
|
62
|
+
when 'GET'
|
63
|
+
conn.get resource, data
|
64
|
+
when 'POST'
|
65
|
+
conn.post resource, data.to_json
|
66
|
+
when 'PUT'
|
67
|
+
conn.put resource, data.to_json
|
68
|
+
when 'DELETE'
|
69
|
+
conn.delete resource, data
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
10
73
|
def self.is_response_success(response)
|
11
|
-
raise
|
74
|
+
raise CnvrgAPIError.new(JSON(response.body)['errors']) if response.status != 200
|
12
75
|
end
|
13
76
|
end
|
14
|
-
end
|
77
|
+
end
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -49,6 +49,7 @@ require 'cnvrg/job_cli'
|
|
49
49
|
require 'cnvrg/job_ssh'
|
50
50
|
require 'cnvrg/connect_job_ssh'
|
51
51
|
require 'cnvrg/api_v2'
|
52
|
+
require 'rubygems/package'
|
52
53
|
|
53
54
|
class Thor
|
54
55
|
module Base
|
@@ -857,7 +858,7 @@ module Cnvrg
|
|
857
858
|
method_option :read, :type => :boolean, :aliases => ["-r", "--read"], :default => false
|
858
859
|
method_option :remote, :type => :boolean, :aliases => ["-h", "--remote"], :default => false
|
859
860
|
method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
|
860
|
-
def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false)
|
861
|
+
def clone_data(dataset_url, only_tree=false, commit=nil, query=nil, read=false, remote=false, flatten: false, relative: false, soft: false, threads: 15)
|
861
862
|
begin
|
862
863
|
verify_logged_in(false)
|
863
864
|
log_start(__method__, args, options)
|
@@ -913,7 +914,7 @@ module Cnvrg
|
|
913
914
|
|
914
915
|
while files['keys'].length > 0
|
915
916
|
Cnvrg::Logger.log_info("download multiple files, #{downloaded_files.size} files downloaded")
|
916
|
-
@files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten)
|
917
|
+
@files.download_multiple_files_s3(files, @dataset.local_path, progressbar: progressbar, read_only: read, flatten: flatten, threads: threads)
|
917
918
|
|
918
919
|
downloaded_files += files['keys'].length
|
919
920
|
files = @files.get_clone_chunk(commit: commit, latest_id: files['latest'])
|
@@ -1846,6 +1847,7 @@ module Cnvrg
|
|
1846
1847
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
|
1847
1848
|
method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
|
1848
1849
|
method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
|
1850
|
+
method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
|
1849
1851
|
def clone(project_url)
|
1850
1852
|
begin
|
1851
1853
|
verify_logged_in(false)
|
@@ -1856,7 +1858,7 @@ module Cnvrg
|
|
1856
1858
|
owner = url_parts[project_index - 1]
|
1857
1859
|
remote = options["remote"] || false
|
1858
1860
|
soft = options["soft"] || false
|
1859
|
-
|
1861
|
+
threads = options[:threads] || Cnvrg::Helpers.parallel_threads
|
1860
1862
|
|
1861
1863
|
response = Cnvrg::API.request("users/#{owner}/projects/#{slug}/get_project", 'GET')
|
1862
1864
|
Cnvrg::CLI.is_response_success(response)
|
@@ -1907,9 +1909,10 @@ module Cnvrg
|
|
1907
1909
|
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
1908
1910
|
log_message("Downloading files", Thor::Shell::Color::BLUE)
|
1909
1911
|
progressbar = @files.create_progressbar(files.size, "Clone Progress")
|
1910
|
-
@files.download_files(files, commit_sha1, progress: progressbar)
|
1912
|
+
@files.download_files(files, commit_sha1, progress: progressbar, threads: threads)
|
1911
1913
|
progressbar.finish
|
1912
1914
|
Project.verify_cnvrgignore_exist(project_name, remote)
|
1915
|
+
Cnvrg::Logger.log_info("Generating idx")
|
1913
1916
|
@project.set_idx(idx)
|
1914
1917
|
log_message("Done")
|
1915
1918
|
log_message("Downloaded #{files.size} files")
|
@@ -2274,13 +2277,16 @@ module Cnvrg
|
|
2274
2277
|
method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
|
2275
2278
|
method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
|
2276
2279
|
method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
|
2280
|
+
method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
|
2277
2281
|
|
2278
|
-
def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true)
|
2282
|
+
def upload(link = false, sync = false, direct = false, ignore_list = "", in_exp = false, force = false, output_dir = "output", job_type = nil, job_slug = nil, suppress_exceptions = true,chunk_size=1000)
|
2279
2283
|
begin
|
2280
2284
|
# we are passing "force" twice.. doesnt really make sense :\\
|
2281
2285
|
verify_logged_in(true)
|
2282
2286
|
log_start(__method__, args, options)
|
2283
2287
|
@project = Project.new(get_project_home)
|
2288
|
+
chunk_size = chunk_size ? chunk_size : options["chunk_size"]
|
2289
|
+
|
2284
2290
|
|
2285
2291
|
# Enable local/experiment exception logging
|
2286
2292
|
suppress_exceptions = suppress_exceptions ? suppress_exceptions : options[:suppress_exceptions]
|
@@ -2396,7 +2402,7 @@ module Cnvrg
|
|
2396
2402
|
:total => (to_upload.size + deleted.size),
|
2397
2403
|
:autofinish => true)
|
2398
2404
|
|
2399
|
-
buffered_errors = @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions)
|
2405
|
+
buffered_errors = @files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar, suppress_exceptions: suppress_exceptions, chunk_size: chunk_size)
|
2400
2406
|
@files.delete_files_from_server(deleted, commit_sha1, suppress_exceptions: suppress_exceptions)
|
2401
2407
|
|
2402
2408
|
progressbar.finish
|
@@ -2970,7 +2976,7 @@ module Cnvrg
|
|
2970
2976
|
method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
|
2971
2977
|
method_option :suppress_exceptions, :type => :boolean, :aliases => ["--suppress-exceptions"], :default => true
|
2972
2978
|
method_option :debug_mode, :type => :boolean, :aliases => ["--debug-mode"], :default => false
|
2973
|
-
method_option :
|
2979
|
+
method_option :chunk_size, :type => :numeric, :aliases => ["--chunk"], :default => 1000
|
2974
2980
|
|
2975
2981
|
def sync(direct = true)
|
2976
2982
|
verify_logged_in(true) if direct
|
@@ -2995,7 +3001,8 @@ module Cnvrg
|
|
2995
3001
|
end
|
2996
3002
|
invoke :upload, [false, true, direct, "",in_exp,options[:force], output_dir, job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
|
2997
3003
|
:ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
|
2998
|
-
:files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
|
3004
|
+
:files => options["files"], :output_dir => output_dir, :job_slug => job_slug, :job_type => job_type, :suppress_exceptions => options["suppress_exceptions"],
|
3005
|
+
:debug_mode => options['debug_mode'], :git_diff => options["git_diff"], :chunk_size => options["chunk_size"]
|
2999
3006
|
|
3000
3007
|
end
|
3001
3008
|
|
@@ -3039,6 +3046,7 @@ module Cnvrg
|
|
3039
3046
|
method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
|
3040
3047
|
method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
|
3041
3048
|
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
|
3049
|
+
method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
|
3042
3050
|
|
3043
3051
|
def run(*cmd)
|
3044
3052
|
verify_logged_in(true)
|
@@ -3073,6 +3081,7 @@ module Cnvrg
|
|
3073
3081
|
email_notification_error = options["notify_on_error"]
|
3074
3082
|
email_notification_success = options["notify_on_success"]
|
3075
3083
|
emails = options["emails"]
|
3084
|
+
wait = options["wait"]
|
3076
3085
|
|
3077
3086
|
if !data.present? and data_query.present?
|
3078
3087
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
@@ -3118,14 +3127,14 @@ module Cnvrg
|
|
3118
3127
|
invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
3119
3128
|
:schedule => schedule, :recurring => recurring, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
|
3120
3129
|
:image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :sync_before_terminate => sync_before_terminate,
|
3121
|
-
|
3130
|
+
:max_time => max_time,
|
3122
3131
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
3123
3132
|
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
3124
3133
|
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
|
3125
|
-
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
|
3134
|
+
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails, :wait => wait
|
3135
|
+
|
3126
3136
|
return
|
3127
3137
|
end
|
3128
|
-
|
3129
3138
|
end
|
3130
3139
|
|
3131
3140
|
desc '', '', :hide => true
|
@@ -3288,7 +3297,7 @@ module Cnvrg
|
|
3288
3297
|
puts line
|
3289
3298
|
end
|
3290
3299
|
log << cur_log
|
3291
|
-
if log.size >=
|
3300
|
+
if log.size >= 10
|
3292
3301
|
@exp.upload_temp_log(log) unless log.empty?
|
3293
3302
|
log = []
|
3294
3303
|
elsif (start_time + 15.seconds) <= Time.now
|
@@ -3445,6 +3454,7 @@ module Cnvrg
|
|
3445
3454
|
method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
|
3446
3455
|
method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
|
3447
3456
|
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
|
3457
|
+
method_option :wait, :type => :boolean, :aliases => ["-w", "--wait"], :default => false, :desc => "keep command session open until experiment finished to return exit status"
|
3448
3458
|
|
3449
3459
|
def exec_remote(*cmd)
|
3450
3460
|
|
@@ -3506,7 +3516,7 @@ module Cnvrg
|
|
3506
3516
|
options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
3507
3517
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
3508
3518
|
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets",
|
3509
|
-
"requirements", "prerun", "email_notification_error", "email_notification_success", "emails")
|
3519
|
+
"requirements", "prerun", "email_notification_error", "email_notification_success", "emails", "wait")
|
3510
3520
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
3511
3521
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
3512
3522
|
commit_to_run = options["commit"] || nil
|
@@ -3584,11 +3594,37 @@ module Cnvrg
|
|
3584
3594
|
end
|
3585
3595
|
|
3586
3596
|
log_message(str, Thor::Shell::Color::GREEN)
|
3597
|
+
|
3598
|
+
exit_status = 0
|
3599
|
+
|
3600
|
+
if options['wait']
|
3601
|
+
while true
|
3602
|
+
tries = 0
|
3603
|
+
begin
|
3604
|
+
result = Cnvrg::API_V2.request(
|
3605
|
+
"#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}/status"
|
3606
|
+
)
|
3607
|
+
exit_status = result["exit_status"]
|
3608
|
+
if exit_status
|
3609
|
+
puts "Experiment was exited with status: #{exit_status}"
|
3610
|
+
break
|
3611
|
+
else
|
3612
|
+
system("clear") || system("cls")
|
3613
|
+
puts "#{Time.current}: waiting for experiment to finish"
|
3614
|
+
sleep 3
|
3615
|
+
end
|
3616
|
+
rescue => e
|
3617
|
+
log_error(e)
|
3618
|
+
log_message("Error occurred, retrying", Thor::Shell::Color::RED)
|
3619
|
+
sleep 3
|
3620
|
+
tries += 1
|
3621
|
+
retry if tries <= 5
|
3622
|
+
end
|
3623
|
+
end
|
3624
|
+
end
|
3587
3625
|
|
3588
|
-
exit(
|
3589
|
-
# end
|
3626
|
+
exit(exit_status.to_i)
|
3590
3627
|
end
|
3591
|
-
|
3592
3628
|
rescue => e
|
3593
3629
|
log_message("Error occurred, Aborting", Thor::Shell::Color::RED)
|
3594
3630
|
log_error(e)
|
@@ -3604,6 +3640,7 @@ module Cnvrg
|
|
3604
3640
|
exit(1)
|
3605
3641
|
end
|
3606
3642
|
end
|
3643
|
+
|
3607
3644
|
desc 'deploy', 'Deploys model to production'
|
3608
3645
|
method_option :small, :type => :boolean, :aliases => ["-s", "--small"], :default => false
|
3609
3646
|
method_option :medium, :type => :boolean, :aliases => ["-m", "--medium"], :default => false
|
data/lib/cnvrg/data.rb
CHANGED
@@ -154,6 +154,7 @@ module Cnvrg
|
|
154
154
|
method_option :relative, :type => :boolean, :aliases => ["-rel", "--relative"], :default => false
|
155
155
|
method_option :flatten, :type => :boolean, :aliases => ["-f", "--flatten"], :default => false
|
156
156
|
method_option :soft, :type => :boolean, :aliases => ["-s", "--soft"], :default => false, :hide => true
|
157
|
+
method_option :threads, :type => :numeric, :aliases => ["--threads"], :default => 15
|
157
158
|
def clone(dataset_url)
|
158
159
|
cli = Cnvrg::CLI.new()
|
159
160
|
only_tree =options[:only_tree]
|
@@ -163,6 +164,7 @@ module Cnvrg
|
|
163
164
|
remote = options[:remote]
|
164
165
|
soft = options[:soft]
|
165
166
|
flatten = options[:flatten]
|
167
|
+
threads = options[:threads]
|
166
168
|
cli.clone_data(
|
167
169
|
dataset_url,
|
168
170
|
only_tree=only_tree,
|
@@ -172,7 +174,8 @@ module Cnvrg
|
|
172
174
|
remote=remote,
|
173
175
|
flatten: flatten,
|
174
176
|
relative: options[:relative],
|
175
|
-
soft: soft
|
177
|
+
soft: soft,
|
178
|
+
threads: threads
|
176
179
|
)
|
177
180
|
end
|
178
181
|
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -1284,11 +1284,11 @@ module Cnvrg
|
|
1284
1284
|
end
|
1285
1285
|
end
|
1286
1286
|
|
1287
|
-
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false)
|
1287
|
+
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false, flatten: false, threads: 15)
|
1288
1288
|
begin
|
1289
1289
|
refresh_storage_token
|
1290
1290
|
parallel_options = {
|
1291
|
-
in_threads:
|
1291
|
+
in_threads: threads,
|
1292
1292
|
isolation: true
|
1293
1293
|
}
|
1294
1294
|
Parallel.map(files["keys"], parallel_options) do |f|
|
data/lib/cnvrg/files.rb
CHANGED
@@ -86,7 +86,7 @@ module Cnvrg
|
|
86
86
|
end
|
87
87
|
end
|
88
88
|
|
89
|
-
def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false)
|
89
|
+
def upload_multiple_files(files_list, commit_sha1, progress: nil, suppress_exceptions: false, chunk_size: 100)
|
90
90
|
#open files on the server.
|
91
91
|
Cnvrg::Logger.log_info("Uploading project files")
|
92
92
|
return if files_list.blank?
|
@@ -95,56 +95,62 @@ module Cnvrg
|
|
95
95
|
return self.upload_files_old(files_list, commit_sha1, progress: progress)
|
96
96
|
end
|
97
97
|
|
98
|
-
|
99
|
-
resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
|
100
|
-
unless Cnvrg::CLI.is_response_success(resp, false)
|
101
|
-
raise SignalException.new("Cant upload files to the server.")
|
102
|
-
end
|
103
|
-
# resolve bucket
|
104
|
-
res = resp['result']
|
105
|
-
files = res['files']
|
106
|
-
|
107
|
-
#upload files
|
108
|
-
token_mutex = Mutex.new
|
98
|
+
blob_ids = []
|
109
99
|
buffered_errors = {}
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
Cnvrg::Logger.log_error(e)
|
121
|
-
Cnvrg::Logger.log_method(bind: binding)
|
122
|
-
|
123
|
-
buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
|
124
|
-
|
125
|
-
raise e unless suppress_exceptions
|
100
|
+
files_list.each_slice(chunk_size).each do |chunk_of_files|
|
101
|
+
Cnvrg::Logger.log_info("Upload chunk")
|
102
|
+
parsed_chunk_of_files = chunk_of_files.map{|x| [x, self.parse_file(x)] if self.parse_file(x)}.compact.to_h
|
103
|
+
|
104
|
+
resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {
|
105
|
+
files: parsed_chunk_of_files,
|
106
|
+
commit: commit_sha1
|
107
|
+
})
|
108
|
+
unless Cnvrg::CLI.is_response_success(resp, false)
|
109
|
+
raise SignalException.new("Cant upload files to the server.")
|
126
110
|
end
|
127
|
-
|
111
|
+
# resolve bucket
|
112
|
+
res = resp['result']
|
113
|
+
files = res['files']
|
114
|
+
|
115
|
+
#upload files
|
116
|
+
blob_id_chunk = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
|
117
|
+
begin
|
118
|
+
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(parsed_chunk_of_files[file]))}
|
119
|
+
rescue => e
|
120
|
+
Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red') unless suppress_exceptions
|
121
|
+
Cnvrg::Logger.log_error(e)
|
122
|
+
Cnvrg::Logger.log_method(bind: binding)
|
123
|
+
|
124
|
+
buffered_errors[file] = "Failed to upload #{file}: #{e.message}" if suppress_exceptions
|
125
|
+
|
126
|
+
raise e unless suppress_exceptions
|
127
|
+
end
|
128
|
+
progress.progress += 1 if progress.present?
|
128
129
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
130
|
+
unless buffered_errors.key?(file)
|
131
|
+
files[file]["bv_id"]
|
132
|
+
else
|
133
|
+
nil
|
134
|
+
end
|
133
135
|
end
|
134
|
-
end
|
135
136
|
|
136
|
-
# remove nil files (failed files) from blob_ids
|
137
|
-
blob_ids.compact!
|
138
137
|
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
138
|
+
blob_ids.concat blob_id_chunk
|
139
|
+
end
|
140
|
+
|
141
|
+
# remove nil files (failed files) from blob_ids
|
142
|
+
blob_ids.compact!
|
144
143
|
|
145
|
-
|
144
|
+
#save files on the server.
|
145
|
+
resp = Cnvrg::API.request(@base_resource + "upload_files_save", 'POST', {blob_ids: blob_ids, commit: commit_sha1})
|
146
|
+
unless Cnvrg::CLI.is_response_success(resp, false)
|
147
|
+
raise SignalException.new("Cant save uploaded files to the server.")
|
146
148
|
end
|
147
149
|
|
150
|
+
return buffered_errors
|
151
|
+
end
|
152
|
+
|
153
|
+
|
148
154
|
|
149
155
|
def delete_files_from_server_old(files, commit_sha1)
|
150
156
|
#files are absolute path here.
|
@@ -717,7 +723,7 @@ module Cnvrg
|
|
717
723
|
@progressbar
|
718
724
|
end
|
719
725
|
|
720
|
-
def download_files(files, commit, postfix: '', progress: nil)
|
726
|
+
def download_files(files, commit, postfix: '', progress: nil, threads: 15)
|
721
727
|
return if files.blank?
|
722
728
|
if Cnvrg::Helpers.server_version < 1
|
723
729
|
Cnvrg::Logger.log_info("Download files from older server.")
|
@@ -727,7 +733,7 @@ module Cnvrg
|
|
727
733
|
unless Cnvrg::CLI.is_response_success(res, false)
|
728
734
|
raise SignalException.new("Cant download files from the server.")
|
729
735
|
end
|
730
|
-
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress)
|
736
|
+
self.download_multpile_files_s3(res['result'], @project_home, postfix: postfix, progress: progress, threads: threads)
|
731
737
|
end
|
732
738
|
|
733
739
|
|
@@ -744,7 +750,7 @@ module Cnvrg
|
|
744
750
|
conflicted.each{|file| self.delete_conflict(file); progress.progress += 1 if progress.present?}
|
745
751
|
end
|
746
752
|
|
747
|
-
def download_multpile_files_s3(files, project_home, postfix: '', progress: nil)
|
753
|
+
def download_multpile_files_s3(files, project_home, postfix: '', progress: nil, threads: 15)
|
748
754
|
begin
|
749
755
|
props = {}
|
750
756
|
client = props[:client]
|
@@ -753,7 +759,7 @@ module Cnvrg
|
|
753
759
|
bucket = props[:bucket]
|
754
760
|
download_succ_count = 0
|
755
761
|
parallel_options = {
|
756
|
-
in_threads:
|
762
|
+
in_threads: threads,
|
757
763
|
isolation: true
|
758
764
|
}
|
759
765
|
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -2,15 +2,8 @@ module Cnvrg
|
|
2
2
|
module Helpers
|
3
3
|
|
4
4
|
extend self
|
5
|
-
def parallel_threads
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
### if empty, default will be 15 threads
|
10
|
-
threads = threads > 0 ? threads : 15
|
11
|
-
|
12
|
-
### set max threads to be 100k
|
13
|
-
[threads, 100000].min
|
5
|
+
def parallel_threads
|
6
|
+
15
|
14
7
|
end
|
15
8
|
|
16
9
|
def self.parallel_options
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.10.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-08-04 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|