cnvrg 1.6.31 → 1.6.38
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +22 -17
- data/lib/cnvrg/data.rb +4 -1
- data/lib/cnvrg/datafiles.rb +215 -201
- data/lib/cnvrg/experiment.rb +3 -2
- data/lib/cnvrg/files.rb +22 -1
- data/lib/cnvrg/project.rb +8 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e5953510af9633c925be99ac1463aef31667eed5f83f5b0c610672ab9fe375cb
|
4
|
+
data.tar.gz: 7bd8fce2d93a20ed695e4c202f2b29092de84bf90332385a3921844046a0c591
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8b93cd899060800095895967dbe2b8f3391e2d77b6f29ea05cd6651de4d26de2ba24fdaedaa8dafad82a22e250d95abf7651d2b77afe2dcf68b883461774c965
|
7
|
+
data.tar.gz: a4dc324e6f58b628013bb00278a74233f6b2c6e4ffa260e9edf74ea2557ca144907449078defb1abdb22ffe411fdde196b2e029bd66e67233140cf54d6caf67d
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1184,7 +1184,7 @@ module Cnvrg
|
|
1184
1184
|
end
|
1185
1185
|
|
1186
1186
|
desc '', '', :hide => true
|
1187
|
-
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
|
1187
|
+
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
|
1188
1188
|
begin
|
1189
1189
|
verify_logged_in(false)
|
1190
1190
|
log_start(__method__, args, options)
|
@@ -1204,7 +1204,7 @@ module Cnvrg
|
|
1204
1204
|
log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
|
1205
1205
|
number_of_chunks = (@files.size.to_f / chunk_size).ceil
|
1206
1206
|
if commit.blank?
|
1207
|
-
response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
|
1207
|
+
response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
|
1208
1208
|
unless response #means we failed in the start commit.
|
1209
1209
|
raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
|
1210
1210
|
end
|
@@ -1837,8 +1837,6 @@ module Cnvrg
|
|
1837
1837
|
end
|
1838
1838
|
clone_resp = Project.clone_dir(slug, owner, project_name,git)
|
1839
1839
|
project_home = Dir.pwd + "/" + project_name
|
1840
|
-
|
1841
|
-
|
1842
1840
|
end
|
1843
1841
|
|
1844
1842
|
if clone_resp
|
@@ -2213,6 +2211,7 @@ module Cnvrg
|
|
2213
2211
|
method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
|
2214
2212
|
method_option :files, :type => :string, :aliases => ["--files"], :default => nil
|
2215
2213
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
2214
|
+
method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
|
2216
2215
|
method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
|
2217
2216
|
method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
|
2218
2217
|
|
@@ -2237,19 +2236,21 @@ module Cnvrg
|
|
2237
2236
|
spec_files_to_upload = spec_files_to_upload.split(",")
|
2238
2237
|
end
|
2239
2238
|
if @project.is_git
|
2239
|
+
list = []
|
2240
2240
|
git_output_dir = options["output_dir"] || output_dir
|
2241
2241
|
if git_output_dir.present?
|
2242
2242
|
if git_output_dir.ends_with? "/"
|
2243
2243
|
git_output_dir = git_output_dir[0..-2]
|
2244
2244
|
end
|
2245
2245
|
list = @project.generate_output_dir(git_output_dir)
|
2246
|
-
spec_files_to_upload = list
|
2247
|
-
if spec_files_to_upload.blank?
|
2248
|
-
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
2249
|
-
return true
|
2250
|
-
end
|
2251
|
-
force = true
|
2252
2246
|
end
|
2247
|
+
list += @project.generate_git_diff if options["git_diff"]
|
2248
|
+
spec_files_to_upload = list
|
2249
|
+
if spec_files_to_upload.blank?
|
2250
|
+
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
2251
|
+
return true
|
2252
|
+
end
|
2253
|
+
force = true
|
2253
2254
|
end
|
2254
2255
|
|
2255
2256
|
if ignore.nil? or ignore.empty?
|
@@ -2894,6 +2895,7 @@ module Cnvrg
|
|
2894
2895
|
method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
|
2895
2896
|
method_option :files, :type => :string, :aliases => ["--files"], :default => nil
|
2896
2897
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
2898
|
+
method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
|
2897
2899
|
def sync(direct = true)
|
2898
2900
|
verify_logged_in(true) if direct
|
2899
2901
|
@project = Project.new(get_project_home)
|
@@ -2914,7 +2916,7 @@ module Cnvrg
|
|
2914
2916
|
end
|
2915
2917
|
invoke :upload, [false, true, direct, "",in_exp,options[:force], options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
|
2916
2918
|
:ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
|
2917
|
-
:files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
|
2919
|
+
:files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type, :git_diff=> options["git_diff"]
|
2918
2920
|
end
|
2919
2921
|
|
2920
2922
|
desc 'run cmd', 'Runs an experiment'
|
@@ -2956,6 +2958,7 @@ module Cnvrg
|
|
2956
2958
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
2957
2959
|
method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
|
2958
2960
|
method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
|
2961
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
|
2959
2962
|
|
2960
2963
|
def run(*cmd)
|
2961
2964
|
verify_logged_in(true)
|
@@ -2989,6 +2992,7 @@ module Cnvrg
|
|
2989
2992
|
requirements = options["requirements"]
|
2990
2993
|
email_notification_error = options["notify_on_error"]
|
2991
2994
|
email_notification_success = options["notify_on_success"]
|
2995
|
+
emails = options["emails"]
|
2992
2996
|
|
2993
2997
|
if !data.present? and data_query.present?
|
2994
2998
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
@@ -3003,7 +3007,6 @@ module Cnvrg
|
|
3003
3007
|
restart_if_stuck = options["restart_if_stuck"]
|
3004
3008
|
|
3005
3009
|
options_hash = Hash[options]
|
3006
|
-
|
3007
3010
|
if local
|
3008
3011
|
if Cnvrg::Helpers.windows?
|
3009
3012
|
say "Windows is currently not supported for running experiments locally"
|
@@ -3039,7 +3042,7 @@ module Cnvrg
|
|
3039
3042
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
3040
3043
|
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
3041
3044
|
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
|
3042
|
-
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success
|
3045
|
+
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
|
3043
3046
|
return
|
3044
3047
|
end
|
3045
3048
|
|
@@ -3345,8 +3348,9 @@ module Cnvrg
|
|
3345
3348
|
method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
|
3346
3349
|
method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
|
3347
3350
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
3348
|
-
method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
|
3349
|
-
method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
|
3351
|
+
method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
|
3352
|
+
method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
|
3353
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
|
3350
3354
|
|
3351
3355
|
def exec_remote(*cmd)
|
3352
3356
|
|
@@ -3368,6 +3372,7 @@ module Cnvrg
|
|
3368
3372
|
requirements = options["requirements"]
|
3369
3373
|
email_notification_error = options["email_notification_error"]
|
3370
3374
|
email_notification_success = options["email_notification_success"]
|
3375
|
+
emails = options["emails"]
|
3371
3376
|
max_time = options["max_time"]
|
3372
3377
|
if !max_time.nil? and !max_time.empty?
|
3373
3378
|
max_time = max_time.to_i
|
@@ -3407,7 +3412,7 @@ module Cnvrg
|
|
3407
3412
|
options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
3408
3413
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
3409
3414
|
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
|
3410
|
-
"email_notification_error", "email_notification_success")
|
3415
|
+
"email_notification_error", "email_notification_success", "emails")
|
3411
3416
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
3412
3417
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
3413
3418
|
commit_to_run = options["commit"] || nil
|
@@ -3475,7 +3480,7 @@ module Cnvrg
|
|
3475
3480
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
3476
3481
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
3477
3482
|
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
|
3478
|
-
email_notification_error: email_notification_error, email_notification_success: email_notification_success)
|
3483
|
+
email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
|
3479
3484
|
if Cnvrg::CLI.is_response_success(res)
|
3480
3485
|
check = Helpers.checkmark()
|
3481
3486
|
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
data/lib/cnvrg/data.rb
CHANGED
@@ -186,11 +186,14 @@ module Cnvrg
|
|
186
186
|
desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
|
187
187
|
method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
|
188
188
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
|
189
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
189
190
|
def put(dataset_url, *files)
|
190
191
|
cli = Cnvrg::CLI.new()
|
191
192
|
dir = options[:dir]
|
192
193
|
commit = options[:commit]
|
193
|
-
|
194
|
+
message = options[:message]
|
195
|
+
|
196
|
+
cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
|
194
197
|
end
|
195
198
|
|
196
199
|
desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -9,8 +9,8 @@ module Cnvrg
|
|
9
9
|
class Datafiles
|
10
10
|
ParallelThreads ||= Cnvrg::Helpers.parallel_threads
|
11
11
|
|
12
|
-
LARGE_FILE=1024*1024*5
|
13
|
-
MULTIPART_SPLIT=10000000
|
12
|
+
LARGE_FILE = 1024 * 1024 * 5
|
13
|
+
MULTIPART_SPLIT = 10000000
|
14
14
|
|
15
15
|
attr_reader :base_resource
|
16
16
|
|
@@ -20,11 +20,20 @@ module Cnvrg
|
|
20
20
|
@dataset = dataset
|
21
21
|
@base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
|
22
22
|
@downloader = @dataset.get_storage_client
|
23
|
+
@token_issue_time = Time.current
|
24
|
+
end
|
25
|
+
|
26
|
+
def refresh_storage_token
|
27
|
+
current_time = Time.current
|
28
|
+
if current_time - @token_issue_time > 3.hours
|
29
|
+
@downloader = @dataset.get_storage_client
|
30
|
+
@token_issue_time = Time.current
|
31
|
+
end
|
23
32
|
end
|
24
33
|
|
25
34
|
def check_file_sha1(filename, org_sha1, tag: 'conflict')
|
26
35
|
file_loc = "#{Dir.pwd}/#{filename}"
|
27
|
-
sha1 =
|
36
|
+
sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
|
28
37
|
return 0 if sha1 == org_sha1
|
29
38
|
FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
|
30
39
|
return 1
|
@@ -43,7 +52,7 @@ module Cnvrg
|
|
43
52
|
end
|
44
53
|
raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
|
45
54
|
end
|
46
|
-
paths.map{|p| p.gsub(/^\.\//, '')}
|
55
|
+
paths.map { |p| p.gsub(/^\.\//, '') }
|
47
56
|
end
|
48
57
|
|
49
58
|
def check_files_sha1(files, resolver, tag)
|
@@ -74,11 +83,11 @@ module Cnvrg
|
|
74
83
|
prefix = @dataset.get_dataset["bucket_prefix"]
|
75
84
|
batch_size = 10000
|
76
85
|
pbar = ProgressBar.create(:title => "Download Progress",
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
86
|
+
:progress_mark => '=',
|
87
|
+
:format => "%b%i| %c Files downloaded",
|
88
|
+
:starting_at => 0,
|
89
|
+
:total => nil,
|
90
|
+
:autofinish => true)
|
82
91
|
parallel_options = {
|
83
92
|
in_threads: ParallelThreads,
|
84
93
|
in_processes: Cnvrg::CLI::ParallelProcesses,
|
@@ -107,7 +116,7 @@ module Cnvrg
|
|
107
116
|
if response.present?
|
108
117
|
msg = response['result']
|
109
118
|
else
|
110
|
-
msg =
|
119
|
+
msg = "Cant save changes in the dataset"
|
111
120
|
end
|
112
121
|
|
113
122
|
Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
|
@@ -117,6 +126,7 @@ module Cnvrg
|
|
117
126
|
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
|
118
127
|
begin
|
119
128
|
Cnvrg::Logger.log_info("Sending Upload Files request")
|
129
|
+
refresh_storage_token
|
120
130
|
error = nil
|
121
131
|
upload_resp = nil
|
122
132
|
10.times do
|
@@ -161,9 +171,9 @@ module Cnvrg
|
|
161
171
|
end
|
162
172
|
@temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
|
163
173
|
end
|
164
|
-
blob_ids = files.values.map {|f| f['bv_id']}
|
174
|
+
blob_ids = files.values.map { |f| f['bv_id'] }
|
165
175
|
if blob_ids.present?
|
166
|
-
dirs = tree.keys.select {|k| tree[k].nil?} || []
|
176
|
+
dirs = tree.keys.select { |k| tree[k].nil? } || []
|
167
177
|
Cnvrg::Logger.info("Sending Upload files save")
|
168
178
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
|
169
179
|
unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
@@ -198,12 +208,12 @@ module Cnvrg
|
|
198
208
|
file_size = File.size(absolute_path).to_f
|
199
209
|
mime_type = MimeMagic.by_path(absolute_path)
|
200
210
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
201
|
-
sha1 =
|
211
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
202
212
|
if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
|
203
213
|
is_valid = false
|
204
214
|
begin
|
205
|
-
|
206
|
-
|
215
|
+
content = open(absolute_path).read()
|
216
|
+
hash = YAML.load(open(absolute_path).read())
|
207
217
|
# if level 1 keys count is 1
|
208
218
|
if hash.keys.count == 1
|
209
219
|
if hash["tags"].present?
|
@@ -211,7 +221,7 @@ module Cnvrg
|
|
211
221
|
elsif hash[hash.keys.first].class != Hash
|
212
222
|
is_valid = true
|
213
223
|
end
|
214
|
-
|
224
|
+
# if level 1 keys count is greater than 1
|
215
225
|
elsif hash.keys.count > 1
|
216
226
|
if hash["tags"].present? and hash["tags"].class == Hash
|
217
227
|
is_valid = false
|
@@ -245,6 +255,7 @@ module Cnvrg
|
|
245
255
|
return s3_res
|
246
256
|
end
|
247
257
|
end
|
258
|
+
|
248
259
|
def upload_tar_file(absolute_path, relative_path, commit_sha1)
|
249
260
|
begin
|
250
261
|
file_name = File.basename relative_path
|
@@ -252,24 +263,24 @@ module Cnvrg
|
|
252
263
|
mime_type = MimeMagic.by_path(absolute_path)
|
253
264
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
254
265
|
begin
|
255
|
-
chunked_bytes = [100, (file_size*0.01)].min
|
266
|
+
chunked_bytes = [100, (file_size * 0.01)].min
|
256
267
|
total_yanked = ""
|
257
268
|
open(absolute_path, "rb") do |f|
|
258
269
|
total_yanked = f.read(chunked_bytes)
|
259
270
|
end
|
260
271
|
if !total_yanked.empty?
|
261
|
-
sha1 =
|
272
|
+
sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
|
262
273
|
else
|
263
|
-
sha1 =
|
274
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
264
275
|
end
|
265
276
|
rescue
|
266
|
-
sha1 =
|
277
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
267
278
|
end
|
268
279
|
|
269
280
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
270
281
|
commit_sha1: commit_sha1, file_name: file_name,
|
271
282
|
file_size: file_size, file_content_type: content_type, sha1: sha1,
|
272
|
-
|
283
|
+
new_version: true})
|
273
284
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
274
285
|
path = upload_resp["result"]["path"]
|
275
286
|
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
@@ -305,14 +316,15 @@ module Cnvrg
|
|
305
316
|
return false
|
306
317
|
|
307
318
|
end
|
308
|
-
|
319
|
+
|
320
|
+
def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
|
309
321
|
file_name = File.basename relative_path
|
310
322
|
file_size = File.size(absolute_path).to_f
|
311
323
|
content_type = "text/x-log"
|
312
324
|
upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
313
|
-
|
314
|
-
|
315
|
-
data_commit_sha1:data_commit_sha})
|
325
|
+
file_name: file_name, log_date: Time.now,
|
326
|
+
file_size: file_size, file_content_type: content_type,
|
327
|
+
data_commit_sha1: data_commit_sha})
|
316
328
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
317
329
|
path = upload_resp["result"]["path"]
|
318
330
|
s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
|
@@ -403,15 +415,15 @@ module Cnvrg
|
|
403
415
|
|
404
416
|
end
|
405
417
|
|
406
|
-
def upload_cnvrg_image(absolute_path, image_name,secret)
|
418
|
+
def upload_cnvrg_image(absolute_path, image_name, secret)
|
407
419
|
file_name = File.basename relative_path
|
408
420
|
file_size = File.size(absolute_path).to_f
|
409
421
|
mime_type = MimeMagic.by_path(absolute_path)
|
410
422
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
411
423
|
upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
|
412
|
-
|
413
|
-
|
414
|
-
|
424
|
+
file_name: file_name,
|
425
|
+
file_size: file_size, file_content_type: content_type,
|
426
|
+
secret: secret})
|
415
427
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
416
428
|
path = upload_resp["result"]["path"]
|
417
429
|
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
@@ -455,7 +467,7 @@ module Cnvrg
|
|
455
467
|
sts_path = upload_resp["result"]["path_sts"]
|
456
468
|
|
457
469
|
retries = 0
|
458
|
-
success= false
|
470
|
+
success = false
|
459
471
|
while !success and retries < 20
|
460
472
|
begin
|
461
473
|
if !Helpers.is_verify_ssl
|
@@ -465,7 +477,7 @@ module Cnvrg
|
|
465
477
|
end
|
466
478
|
success = true
|
467
479
|
rescue => e
|
468
|
-
retries +=1
|
480
|
+
retries += 1
|
469
481
|
sleep(5)
|
470
482
|
|
471
483
|
end
|
@@ -477,33 +489,33 @@ module Cnvrg
|
|
477
489
|
key = split[0]
|
478
490
|
iv = split[1]
|
479
491
|
is_s3 = upload_resp["result"]["is_s3"]
|
480
|
-
access =
|
492
|
+
access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
|
481
493
|
|
482
|
-
secret =
|
494
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
|
483
495
|
|
484
|
-
session =
|
485
|
-
region =
|
496
|
+
session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
|
497
|
+
region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
|
486
498
|
|
487
|
-
bucket =
|
488
|
-
server_side_encryption =upload_resp["result"]["server_side_encryption"]
|
499
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
|
500
|
+
server_side_encryption = upload_resp["result"]["server_side_encryption"]
|
489
501
|
use_accelerate_endpoint = false
|
490
502
|
if is_s3 or is_s3.nil?
|
491
503
|
|
492
|
-
use_accelerate_endpoint =true
|
504
|
+
use_accelerate_endpoint = true
|
493
505
|
client = Aws::S3::Client.new(
|
494
|
-
:access_key_id =>access,
|
506
|
+
:access_key_id => access,
|
495
507
|
:secret_access_key => secret,
|
496
508
|
:session_token => session,
|
497
509
|
:region => region,
|
498
510
|
:http_open_timeout => 60, :retry_limit => 20)
|
499
511
|
else
|
500
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
|
512
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
|
501
513
|
use_accelerate_endpoint = false
|
502
514
|
client = Aws::S3::Client.new(
|
503
|
-
:access_key_id =>access,
|
515
|
+
:access_key_id => access,
|
504
516
|
:secret_access_key => secret,
|
505
517
|
:region => region,
|
506
|
-
:endpoint=> endpoint
|
518
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
507
519
|
:http_open_timeout => 60, :retry_limit => 20)
|
508
520
|
end
|
509
521
|
|
@@ -515,13 +527,11 @@ module Cnvrg
|
|
515
527
|
end
|
516
528
|
|
517
529
|
|
518
|
-
|
519
|
-
|
520
530
|
s3 = Aws::S3::Resource.new(client: client)
|
521
531
|
|
522
532
|
resp = s3.bucket(bucket).
|
523
|
-
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
524
|
-
upload_file(file_path,options)
|
533
|
+
object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
|
534
|
+
upload_file(file_path, options)
|
525
535
|
|
526
536
|
|
527
537
|
return resp
|
@@ -578,6 +588,7 @@ module Cnvrg
|
|
578
588
|
response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
579
589
|
return Cnvrg::CLI.is_response_success(response, false)
|
580
590
|
end
|
591
|
+
|
581
592
|
def download_list_files_in_query(response, dataset_home)
|
582
593
|
sts_path = response["path_sts"]
|
583
594
|
if !Helpers.is_verify_ssl
|
@@ -589,29 +600,29 @@ module Cnvrg
|
|
589
600
|
key = split[0]
|
590
601
|
iv = split[1]
|
591
602
|
|
592
|
-
access =
|
603
|
+
access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
|
593
604
|
|
594
|
-
secret =
|
605
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
|
595
606
|
|
596
|
-
session =
|
597
|
-
region =
|
607
|
+
session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
|
608
|
+
region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
|
598
609
|
|
599
|
-
bucket =
|
610
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
|
600
611
|
is_s3 = response["is_s3"]
|
601
612
|
if is_s3 or is_s3.nil?
|
602
613
|
client = Aws::S3::Client.new(
|
603
|
-
:access_key_id =>access,
|
614
|
+
:access_key_id => access,
|
604
615
|
:secret_access_key => secret,
|
605
616
|
:session_token => session,
|
606
617
|
:region => region,
|
607
618
|
:http_open_timeout => 60, :retry_limit => 20)
|
608
619
|
else
|
609
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
|
620
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
|
610
621
|
client = Aws::S3::Client.new(
|
611
|
-
:access_key_id =>access,
|
622
|
+
:access_key_id => access,
|
612
623
|
:secret_access_key => secret,
|
613
624
|
:region => region,
|
614
|
-
:endpoint=> endpoint
|
625
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
615
626
|
:http_open_timeout => 60, :retry_limit => 20)
|
616
627
|
end
|
617
628
|
list_files = response["files"]
|
@@ -628,18 +639,18 @@ module Cnvrg
|
|
628
639
|
}
|
629
640
|
download_count = 0
|
630
641
|
Parallel.map((list_files), parallel_options) do |f|
|
631
|
-
file_key =
|
632
|
-
begin
|
642
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
|
633
643
|
begin
|
634
|
-
|
635
|
-
|
636
|
-
|
644
|
+
begin
|
645
|
+
dir = File.dirname f["fullpath"]
|
646
|
+
FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
|
647
|
+
end
|
637
648
|
|
638
|
-
|
639
|
-
|
640
|
-
|
641
|
-
|
642
|
-
|
649
|
+
File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
|
650
|
+
resp = client.get_object({bucket: bucket,
|
651
|
+
key: file_key}, target: file)
|
652
|
+
end
|
653
|
+
download_count += 1
|
643
654
|
rescue
|
644
655
|
end
|
645
656
|
|
@@ -651,97 +662,95 @@ module Cnvrg
|
|
651
662
|
end
|
652
663
|
|
653
664
|
|
654
|
-
|
655
|
-
|
656
665
|
end
|
657
666
|
|
658
667
|
|
659
|
-
def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
|
668
|
+
def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
|
660
669
|
begin
|
661
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1
|
670
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
|
662
671
|
Cnvrg::CLI.is_response_success(res, false)
|
663
|
-
|
664
|
-
|
672
|
+
if res["result"]
|
673
|
+
file_url = res["result"]["file_url"]
|
665
674
|
|
666
|
-
|
667
|
-
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
|
672
|
-
|
673
|
-
|
674
|
-
|
675
|
-
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
686
|
-
absolute_path += ".conflict" if conflict
|
687
|
-
sts_path = download_resp["result"]["path_sts"]
|
688
|
-
retries = 0
|
689
|
-
success= false
|
690
|
-
while !success and retries < 20
|
691
|
-
begin
|
692
|
-
if !Helpers.is_verify_ssl
|
693
|
-
body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
|
694
|
-
else
|
695
|
-
body = open(sts_path).read
|
696
|
-
end
|
697
|
-
success = true
|
698
|
-
rescue => e
|
699
|
-
retries +=1
|
700
|
-
sleep(5)
|
675
|
+
if as_link
|
676
|
+
return res["result"]
|
677
|
+
end
|
678
|
+
# begin
|
679
|
+
# if !Helpers.is_verify_ssl
|
680
|
+
# tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
|
681
|
+
#
|
682
|
+
# else
|
683
|
+
# tempfile = Down.download(file_url,open_timeout: 60)
|
684
|
+
#
|
685
|
+
# end
|
686
|
+
#
|
687
|
+
# FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
|
688
|
+
# return true
|
689
|
+
# rescue
|
690
|
+
#
|
691
|
+
# end
|
692
|
+
download_resp = res
|
693
|
+
filename = download_resp["result"]["filename"]
|
701
694
|
|
695
|
+
absolute_path += ".conflict" if conflict
|
696
|
+
sts_path = download_resp["result"]["path_sts"]
|
697
|
+
retries = 0
|
698
|
+
success = false
|
699
|
+
while !success and retries < 20
|
700
|
+
begin
|
701
|
+
if !Helpers.is_verify_ssl
|
702
|
+
body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
|
703
|
+
else
|
704
|
+
body = open(sts_path).read
|
702
705
|
end
|
706
|
+
success = true
|
707
|
+
rescue => e
|
708
|
+
retries += 1
|
709
|
+
sleep(5)
|
710
|
+
|
703
711
|
end
|
704
|
-
|
705
|
-
|
706
|
-
|
707
|
-
|
708
|
-
|
709
|
-
|
712
|
+
end
|
713
|
+
if !success
|
714
|
+
return false
|
715
|
+
end
|
716
|
+
split = body.split("\n")
|
717
|
+
key = split[0]
|
718
|
+
iv = split[1]
|
710
719
|
|
711
|
-
|
720
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
712
721
|
|
713
|
-
|
722
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
|
714
723
|
|
715
|
-
|
716
|
-
|
724
|
+
session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
|
725
|
+
region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
|
717
726
|
|
718
|
-
|
719
|
-
|
727
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
|
728
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
|
720
729
|
|
721
|
-
|
722
|
-
|
723
|
-
|
724
|
-
|
725
|
-
|
726
|
-
|
727
|
-
|
728
|
-
|
729
|
-
|
730
|
-
|
731
|
-
|
732
|
-
|
733
|
-
|
734
|
-
|
735
|
-
|
736
|
-
|
737
|
-
|
730
|
+
is_s3 = download_resp["result"]["is_s3"]
|
731
|
+
if is_s3 or is_s3.nil?
|
732
|
+
client = Aws::S3::Client.new(
|
733
|
+
:access_key_id => access,
|
734
|
+
:secret_access_key => secret,
|
735
|
+
:session_token => session,
|
736
|
+
:region => region,
|
737
|
+
:http_open_timeout => 60, :retry_limit => 20)
|
738
|
+
else
|
739
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
|
740
|
+
client = Aws::S3::Client.new(
|
741
|
+
:access_key_id => access,
|
742
|
+
:secret_access_key => secret,
|
743
|
+
:region => region,
|
744
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
745
|
+
:http_open_timeout => 60, :retry_limit => 20)
|
746
|
+
end
|
738
747
|
|
739
|
-
|
740
|
-
|
741
|
-
|
742
|
-
end
|
743
|
-
return true
|
748
|
+
File.open(project_home + "/" + absolute_path, 'w+') do |file|
|
749
|
+
resp = client.get_object({bucket: bucket,
|
750
|
+
key: file_key}, target: file)
|
744
751
|
end
|
752
|
+
return true
|
753
|
+
end
|
745
754
|
|
746
755
|
rescue => e
|
747
756
|
return false
|
@@ -751,7 +760,7 @@ module Cnvrg
|
|
751
760
|
|
752
761
|
def download_data_file(commit_sha1, dataset_home)
|
753
762
|
begin
|
754
|
-
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
|
763
|
+
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
|
755
764
|
Cnvrg::CLI.is_response_success(res, false)
|
756
765
|
if res["result"]
|
757
766
|
download_resp = res
|
@@ -768,38 +777,38 @@ module Cnvrg
|
|
768
777
|
iv = split[1]
|
769
778
|
|
770
779
|
|
771
|
-
access =
|
780
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
772
781
|
|
773
|
-
secret =
|
782
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
|
774
783
|
|
775
|
-
session =
|
776
|
-
region =
|
784
|
+
session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
|
785
|
+
region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
|
777
786
|
|
778
|
-
bucket =
|
779
|
-
file_key =
|
787
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
|
788
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
|
780
789
|
|
781
790
|
is_s3 = download_resp["result"]["is_s3"]
|
782
791
|
if is_s3 or is_s3.nil?
|
783
792
|
client = Aws::S3::Client.new(
|
784
|
-
:access_key_id =>access,
|
793
|
+
:access_key_id => access,
|
785
794
|
:secret_access_key => secret,
|
786
795
|
:session_token => session,
|
787
796
|
:region => region,
|
788
797
|
:http_open_timeout => 60, :retry_limit => 20)
|
789
798
|
else
|
790
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
|
799
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
|
791
800
|
client = Aws::S3::Client.new(
|
792
|
-
:access_key_id =>access,
|
801
|
+
:access_key_id => access,
|
793
802
|
:secret_access_key => secret,
|
794
803
|
:region => region,
|
795
|
-
:endpoint=> endpoint
|
804
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
796
805
|
:http_open_timeout => 60, :retry_limit => 20)
|
797
806
|
end
|
798
807
|
|
799
808
|
|
800
|
-
File.open(dataset_home+"/"+filename, 'w+') do |file|
|
809
|
+
File.open(dataset_home + "/" + filename, 'w+') do |file|
|
801
810
|
resp = client.get_object({bucket: bucket,
|
802
|
-
|
811
|
+
key: file_key}, target: file)
|
803
812
|
end
|
804
813
|
return filename
|
805
814
|
end
|
@@ -810,7 +819,7 @@ module Cnvrg
|
|
810
819
|
end
|
811
820
|
end
|
812
821
|
|
813
|
-
def download_file(absolute_path, relative_path, project_home, conflict=false)
|
822
|
+
def download_file(absolute_path, relative_path, project_home, conflict = false)
|
814
823
|
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
815
824
|
Cnvrg::CLI.is_response_success(res, false)
|
816
825
|
if res["result"]
|
@@ -848,7 +857,7 @@ module Cnvrg
|
|
848
857
|
puts FileUtils.rmtree("#{absolute_path}")
|
849
858
|
end
|
850
859
|
|
851
|
-
def revoke_download_file(absolute_path, filename, conflict=false)
|
860
|
+
def revoke_download_file(absolute_path, filename, conflict = false)
|
852
861
|
begin
|
853
862
|
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
854
863
|
|
@@ -859,6 +868,7 @@ module Cnvrg
|
|
859
868
|
return false
|
860
869
|
end
|
861
870
|
end
|
871
|
+
|
862
872
|
def revoke_download(tar_files, extracted_files)
|
863
873
|
begin
|
864
874
|
|
@@ -872,32 +882,35 @@ module Cnvrg
|
|
872
882
|
return true
|
873
883
|
|
874
884
|
end
|
885
|
+
|
875
886
|
def delete_commit(commit_sha1)
|
876
887
|
response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
|
877
888
|
Cnvrg::CLI.is_response_success(response, true)
|
878
889
|
return response
|
879
890
|
end
|
891
|
+
|
880
892
|
def get_commit(commit_sha1)
|
881
893
|
response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
|
882
894
|
Cnvrg::CLI.is_response_success(response, true)
|
883
895
|
return response
|
884
896
|
end
|
885
897
|
|
886
|
-
def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
|
898
|
+
def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
|
887
899
|
begin
|
888
900
|
#if we are pushing with force or to branch we dont need to send current/next commit cause we want to
|
889
901
|
# create a new commit.
|
890
902
|
idx = (force || new_branch) ? {} : dataset.get_idx
|
891
903
|
commit = idx[:commit]
|
892
904
|
next_commit = idx[:next_commit]
|
893
|
-
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
|
894
|
-
username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
|
905
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
|
906
|
+
username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
|
895
907
|
Cnvrg::CLI.is_response_success(response, true)
|
896
908
|
return response
|
897
909
|
rescue => e
|
898
910
|
return false
|
899
911
|
end
|
900
912
|
end
|
913
|
+
|
901
914
|
def last_valid_commit()
|
902
915
|
begin
|
903
916
|
#if we are pushing with force or to branch we dont need to send current/next commit cause we want to
|
@@ -910,9 +923,9 @@ module Cnvrg
|
|
910
923
|
end
|
911
924
|
end
|
912
925
|
|
913
|
-
def end_commit(commit_sha1,force, success: true, uploaded_files: 0
|
926
|
+
def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
|
914
927
|
begin
|
915
|
-
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
|
928
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
|
916
929
|
Cnvrg::CLI.is_response_success(response, true)
|
917
930
|
return response
|
918
931
|
rescue => e
|
@@ -938,13 +951,13 @@ module Cnvrg
|
|
938
951
|
end
|
939
952
|
|
940
953
|
def get_trees(commit: "latest")
|
941
|
-
response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
|
954
|
+
response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
|
942
955
|
return nil unless Cnvrg::CLI.is_response_success(response, false)
|
943
956
|
response['result']['files']
|
944
957
|
end
|
945
958
|
|
946
959
|
def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
|
947
|
-
response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
|
960
|
+
response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
|
948
961
|
unless Cnvrg::CLI.is_response_success(response, false)
|
949
962
|
Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
|
950
963
|
return nil
|
@@ -975,26 +988,26 @@ module Cnvrg
|
|
975
988
|
end
|
976
989
|
end
|
977
990
|
|
978
|
-
def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
|
991
|
+
def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
|
979
992
|
begin
|
980
|
-
|
981
|
-
|
982
|
-
|
983
|
-
|
984
|
-
|
985
|
-
|
986
|
-
|
987
|
-
|
988
|
-
|
989
|
-
|
990
|
-
|
991
|
-
|
992
|
-
|
993
|
-
|
994
|
-
|
993
|
+
last_chunk_size = chunk_size
|
994
|
+
q = {commit: commit, chunk_size: chunk_size}
|
995
|
+
overall = 0
|
996
|
+
while last_chunk_size > 0
|
997
|
+
response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
|
998
|
+
if Cnvrg::CLI.is_response_success(response, false)
|
999
|
+
files = response['files']
|
1000
|
+
data_home = "#{Dir.pwd}/#{response['name']}"
|
1001
|
+
last_chunk_size = files['keys'].length
|
1002
|
+
break if last_chunk_size == 0
|
1003
|
+
res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
|
1004
|
+
overall += last_chunk_size
|
1005
|
+
q[:latest] = files['latest']
|
1006
|
+
else
|
1007
|
+
last_chunk_size = 0
|
1008
|
+
end
|
995
1009
|
end
|
996
|
-
|
997
|
-
Cnvrg::Result.new(true, "Cloned #{overall} files!")
|
1010
|
+
Cnvrg::Result.new(true, "Cloned #{overall} files!")
|
998
1011
|
rescue => e
|
999
1012
|
Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
|
1000
1013
|
|
@@ -1002,42 +1015,43 @@ module Cnvrg
|
|
1002
1015
|
end
|
1003
1016
|
|
1004
1017
|
def generate_parallel_idx
|
1005
|
-
|
1018
|
+
|
1006
1019
|
end
|
1007
1020
|
|
1008
|
-
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
|
1021
|
+
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
|
1009
1022
|
begin
|
1023
|
+
refresh_storage_token
|
1010
1024
|
parallel_options = {
|
1011
1025
|
in_threads: ParallelThreads,
|
1012
1026
|
isolation: true
|
1013
1027
|
}
|
1014
1028
|
Parallel.map(files["keys"], parallel_options) do |f|
|
1015
1029
|
begin
|
1016
|
-
|
1017
|
-
|
1018
|
-
|
1019
|
-
|
1020
|
-
|
1021
|
-
|
1022
|
-
|
1030
|
+
local_path = @dataset.local_path + '/' + f['name']
|
1031
|
+
Cnvrg::Logger.log_info("Downloading #{local_path}")
|
1032
|
+
progressbar.progress += 1 if progressbar.present?
|
1033
|
+
if local_path.end_with? "/"
|
1034
|
+
@downloader.mkdir(local_path, recursive: true)
|
1035
|
+
next
|
1036
|
+
end
|
1023
1037
|
# blob
|
1024
|
-
|
1025
|
-
|
1026
|
-
|
1027
|
-
|
1028
|
-
|
1029
|
-
|
1030
|
-
|
1031
|
-
|
1038
|
+
local_path = "#{local_path}.conflict" if conflict
|
1039
|
+
storage_path = f["path"]
|
1040
|
+
if File.exists? local_path
|
1041
|
+
Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
|
1042
|
+
next
|
1043
|
+
end
|
1044
|
+
resp = @downloader.download(storage_path, local_path)
|
1045
|
+
Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
|
1032
1046
|
rescue => e
|
1033
1047
|
Cnvrg::Logger.log_error(e)
|
1034
1048
|
end
|
1035
1049
|
end
|
1036
|
-
return Cnvrg::Result.new(true,"Downloaded successfully")
|
1050
|
+
return Cnvrg::Result.new(true, "Downloaded successfully")
|
1037
1051
|
rescue => e
|
1038
|
-
|
1039
|
-
|
1040
|
-
end
|
1052
|
+
Cnvrg::Logger.log_error(e)
|
1053
|
+
return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
|
1041
1054
|
end
|
1055
|
+
end
|
1042
1056
|
end
|
1043
1057
|
end
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -108,7 +108,7 @@ module Cnvrg
|
|
108
108
|
def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
|
109
109
|
sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
|
110
110
|
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
|
111
|
-
email_notification_error: false, email_notification_success: false)
|
111
|
+
email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
|
112
112
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
113
113
|
commit_sha1: commit_to_run,
|
114
114
|
instance_type: instance_type,
|
@@ -124,7 +124,8 @@ module Cnvrg
|
|
124
124
|
restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
|
125
125
|
prerun: prerun, requirements: requirements, recurring: recurring,
|
126
126
|
email_notification_error: email_notification_error,
|
127
|
-
email_notification_success: email_notification_success
|
127
|
+
email_notification_success: email_notification_success,
|
128
|
+
emails_to_notify: emails_to_notify})
|
128
129
|
|
129
130
|
return response
|
130
131
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -7,7 +7,7 @@ require 'cnvrg/result'
|
|
7
7
|
module Cnvrg
|
8
8
|
class Files
|
9
9
|
ParallelThreads = Cnvrg::Helpers.parallel_threads
|
10
|
-
VALID_FILE_NAME = /[\x00
|
10
|
+
VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
|
11
11
|
LARGE_FILE=1024*1024*5
|
12
12
|
MULTIPART_SPLIT=10000000
|
13
13
|
|
@@ -27,8 +27,16 @@ module Cnvrg
|
|
27
27
|
@custom_progess = false
|
28
28
|
@cli = cli
|
29
29
|
@options = options
|
30
|
+
@token_issue_time = Time.current
|
30
31
|
end
|
31
32
|
|
33
|
+
def refresh_storage_token
|
34
|
+
current_time = Time.current
|
35
|
+
if current_time - @token_issue_time > 3.hours
|
36
|
+
@client = @project.get_storage_client
|
37
|
+
@token_issue_time = Time.current
|
38
|
+
end
|
39
|
+
end
|
32
40
|
|
33
41
|
def self.valid_file_name?(fullpath)
|
34
42
|
VALID_FILE_NAME.match(fullpath).blank?
|
@@ -97,7 +105,13 @@ module Cnvrg
|
|
97
105
|
files = res['files']
|
98
106
|
|
99
107
|
#upload files
|
108
|
+
token_mutex = Mutex.new
|
100
109
|
blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
|
110
|
+
|
111
|
+
token_mutex.synchronize {
|
112
|
+
refresh_storage_token
|
113
|
+
}
|
114
|
+
|
101
115
|
begin
|
102
116
|
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
|
103
117
|
rescue => e
|
@@ -721,8 +735,15 @@ module Cnvrg
|
|
721
735
|
in_threads: Cnvrg::Helpers.parallel_threads,
|
722
736
|
isolation: true
|
723
737
|
}
|
738
|
+
|
739
|
+
token_mutex = Mutex.new
|
740
|
+
|
724
741
|
Parallel.map(files["keys"], parallel_options) do |f|
|
725
742
|
|
743
|
+
token_mutex.synchronize {
|
744
|
+
refresh_storage_token
|
745
|
+
}
|
746
|
+
|
726
747
|
file_path = f["name"]
|
727
748
|
if file_path.end_with? "/"
|
728
749
|
# dir
|
data/lib/cnvrg/project.rb
CHANGED
@@ -370,6 +370,13 @@ module Cnvrg
|
|
370
370
|
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
|
371
371
|
end
|
372
372
|
|
373
|
+
def generate_git_diff
|
374
|
+
git_diff = `git diff --name-only`
|
375
|
+
git_diff.split("\n")
|
376
|
+
rescue
|
377
|
+
[]
|
378
|
+
end
|
379
|
+
|
373
380
|
def generate_output_dir(output_dir)
|
374
381
|
Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
|
375
382
|
upload_list = []
|
@@ -387,7 +394,7 @@ module Cnvrg
|
|
387
394
|
upload_list << output_dir + "/"
|
388
395
|
end
|
389
396
|
Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
|
390
|
-
|
397
|
+
upload_list
|
391
398
|
end
|
392
399
|
def generate_output_dir_tmp(output_dir)
|
393
400
|
upload_list = []
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.38
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-
|
13
|
+
date: 2020-06-23 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|