cnvrg 1.6.31 → 1.6.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +22 -17
- data/lib/cnvrg/data.rb +4 -1
- data/lib/cnvrg/datafiles.rb +215 -201
- data/lib/cnvrg/experiment.rb +3 -2
- data/lib/cnvrg/files.rb +22 -1
- data/lib/cnvrg/project.rb +8 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e5953510af9633c925be99ac1463aef31667eed5f83f5b0c610672ab9fe375cb
|
|
4
|
+
data.tar.gz: 7bd8fce2d93a20ed695e4c202f2b29092de84bf90332385a3921844046a0c591
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8b93cd899060800095895967dbe2b8f3391e2d77b6f29ea05cd6651de4d26de2ba24fdaedaa8dafad82a22e250d95abf7651d2b77afe2dcf68b883461774c965
|
|
7
|
+
data.tar.gz: a4dc324e6f58b628013bb00278a74233f6b2c6e4ffa260e9edf74ea2557ca144907449078defb1abdb22ffe411fdde196b2e029bd66e67233140cf54d6caf67d
|
data/lib/cnvrg/cli.rb
CHANGED
|
@@ -1184,7 +1184,7 @@ module Cnvrg
|
|
|
1184
1184
|
end
|
|
1185
1185
|
|
|
1186
1186
|
desc '', '', :hide => true
|
|
1187
|
-
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
|
|
1187
|
+
def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
|
|
1188
1188
|
begin
|
|
1189
1189
|
verify_logged_in(false)
|
|
1190
1190
|
log_start(__method__, args, options)
|
|
@@ -1204,7 +1204,7 @@ module Cnvrg
|
|
|
1204
1204
|
log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
|
|
1205
1205
|
number_of_chunks = (@files.size.to_f / chunk_size).ceil
|
|
1206
1206
|
if commit.blank?
|
|
1207
|
-
response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
|
|
1207
|
+
response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
|
|
1208
1208
|
unless response #means we failed in the start commit.
|
|
1209
1209
|
raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
|
|
1210
1210
|
end
|
|
@@ -1837,8 +1837,6 @@ module Cnvrg
|
|
|
1837
1837
|
end
|
|
1838
1838
|
clone_resp = Project.clone_dir(slug, owner, project_name,git)
|
|
1839
1839
|
project_home = Dir.pwd + "/" + project_name
|
|
1840
|
-
|
|
1841
|
-
|
|
1842
1840
|
end
|
|
1843
1841
|
|
|
1844
1842
|
if clone_resp
|
|
@@ -2213,6 +2211,7 @@ module Cnvrg
|
|
|
2213
2211
|
method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
|
|
2214
2212
|
method_option :files, :type => :string, :aliases => ["--files"], :default => nil
|
|
2215
2213
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
|
2214
|
+
method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
|
|
2216
2215
|
method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
|
|
2217
2216
|
method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
|
|
2218
2217
|
|
|
@@ -2237,19 +2236,21 @@ module Cnvrg
|
|
|
2237
2236
|
spec_files_to_upload = spec_files_to_upload.split(",")
|
|
2238
2237
|
end
|
|
2239
2238
|
if @project.is_git
|
|
2239
|
+
list = []
|
|
2240
2240
|
git_output_dir = options["output_dir"] || output_dir
|
|
2241
2241
|
if git_output_dir.present?
|
|
2242
2242
|
if git_output_dir.ends_with? "/"
|
|
2243
2243
|
git_output_dir = git_output_dir[0..-2]
|
|
2244
2244
|
end
|
|
2245
2245
|
list = @project.generate_output_dir(git_output_dir)
|
|
2246
|
-
spec_files_to_upload = list
|
|
2247
|
-
if spec_files_to_upload.blank?
|
|
2248
|
-
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
|
2249
|
-
return true
|
|
2250
|
-
end
|
|
2251
|
-
force = true
|
|
2252
2246
|
end
|
|
2247
|
+
list += @project.generate_git_diff if options["git_diff"]
|
|
2248
|
+
spec_files_to_upload = list
|
|
2249
|
+
if spec_files_to_upload.blank?
|
|
2250
|
+
log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
|
|
2251
|
+
return true
|
|
2252
|
+
end
|
|
2253
|
+
force = true
|
|
2253
2254
|
end
|
|
2254
2255
|
|
|
2255
2256
|
if ignore.nil? or ignore.empty?
|
|
@@ -2894,6 +2895,7 @@ module Cnvrg
|
|
|
2894
2895
|
method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
|
|
2895
2896
|
method_option :files, :type => :string, :aliases => ["--files"], :default => nil
|
|
2896
2897
|
method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
|
|
2898
|
+
method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
|
|
2897
2899
|
def sync(direct = true)
|
|
2898
2900
|
verify_logged_in(true) if direct
|
|
2899
2901
|
@project = Project.new(get_project_home)
|
|
@@ -2914,7 +2916,7 @@ module Cnvrg
|
|
|
2914
2916
|
end
|
|
2915
2917
|
invoke :upload, [false, true, direct, "",in_exp,options[:force], options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
|
|
2916
2918
|
:ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
|
|
2917
|
-
:files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
|
|
2919
|
+
:files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type, :git_diff=> options["git_diff"]
|
|
2918
2920
|
end
|
|
2919
2921
|
|
|
2920
2922
|
desc 'run cmd', 'Runs an experiment'
|
|
@@ -2956,6 +2958,7 @@ module Cnvrg
|
|
|
2956
2958
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
|
2957
2959
|
method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
|
|
2958
2960
|
method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
|
|
2961
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
|
|
2959
2962
|
|
|
2960
2963
|
def run(*cmd)
|
|
2961
2964
|
verify_logged_in(true)
|
|
@@ -2989,6 +2992,7 @@ module Cnvrg
|
|
|
2989
2992
|
requirements = options["requirements"]
|
|
2990
2993
|
email_notification_error = options["notify_on_error"]
|
|
2991
2994
|
email_notification_success = options["notify_on_success"]
|
|
2995
|
+
emails = options["emails"]
|
|
2992
2996
|
|
|
2993
2997
|
if !data.present? and data_query.present?
|
|
2994
2998
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
|
@@ -3003,7 +3007,6 @@ module Cnvrg
|
|
|
3003
3007
|
restart_if_stuck = options["restart_if_stuck"]
|
|
3004
3008
|
|
|
3005
3009
|
options_hash = Hash[options]
|
|
3006
|
-
|
|
3007
3010
|
if local
|
|
3008
3011
|
if Cnvrg::Helpers.windows?
|
|
3009
3012
|
say "Windows is currently not supported for running experiments locally"
|
|
@@ -3039,7 +3042,7 @@ module Cnvrg
|
|
|
3039
3042
|
:periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
|
|
3040
3043
|
:output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
|
|
3041
3044
|
:restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
|
|
3042
|
-
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success
|
|
3045
|
+
:email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
|
|
3043
3046
|
return
|
|
3044
3047
|
end
|
|
3045
3048
|
|
|
@@ -3345,8 +3348,9 @@ module Cnvrg
|
|
|
3345
3348
|
method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
|
|
3346
3349
|
method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
|
|
3347
3350
|
method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
|
|
3348
|
-
method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
|
|
3349
|
-
method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
|
|
3351
|
+
method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
|
|
3352
|
+
method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
|
|
3353
|
+
method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
|
|
3350
3354
|
|
|
3351
3355
|
def exec_remote(*cmd)
|
|
3352
3356
|
|
|
@@ -3368,6 +3372,7 @@ module Cnvrg
|
|
|
3368
3372
|
requirements = options["requirements"]
|
|
3369
3373
|
email_notification_error = options["email_notification_error"]
|
|
3370
3374
|
email_notification_success = options["email_notification_success"]
|
|
3375
|
+
emails = options["emails"]
|
|
3371
3376
|
max_time = options["max_time"]
|
|
3372
3377
|
if !max_time.nil? and !max_time.empty?
|
|
3373
3378
|
max_time = max_time.to_i
|
|
@@ -3407,7 +3412,7 @@ module Cnvrg
|
|
|
3407
3412
|
options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
|
|
3408
3413
|
"local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
|
|
3409
3414
|
"data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
|
|
3410
|
-
"email_notification_error", "email_notification_success")
|
|
3415
|
+
"email_notification_error", "email_notification_success", "emails")
|
|
3411
3416
|
exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
|
|
3412
3417
|
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
|
3413
3418
|
commit_to_run = options["commit"] || nil
|
|
@@ -3475,7 +3480,7 @@ module Cnvrg
|
|
|
3475
3480
|
res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
|
|
3476
3481
|
periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
|
|
3477
3482
|
data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
|
|
3478
|
-
email_notification_error: email_notification_error, email_notification_success: email_notification_success)
|
|
3483
|
+
email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
|
|
3479
3484
|
if Cnvrg::CLI.is_response_success(res)
|
|
3480
3485
|
check = Helpers.checkmark()
|
|
3481
3486
|
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
data/lib/cnvrg/data.rb
CHANGED
|
@@ -186,11 +186,14 @@ module Cnvrg
|
|
|
186
186
|
desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
|
|
187
187
|
method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
|
|
188
188
|
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
|
|
189
|
+
method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
|
|
189
190
|
def put(dataset_url, *files)
|
|
190
191
|
cli = Cnvrg::CLI.new()
|
|
191
192
|
dir = options[:dir]
|
|
192
193
|
commit = options[:commit]
|
|
193
|
-
|
|
194
|
+
message = options[:message]
|
|
195
|
+
|
|
196
|
+
cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
|
|
194
197
|
end
|
|
195
198
|
|
|
196
199
|
desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
|
data/lib/cnvrg/datafiles.rb
CHANGED
|
@@ -9,8 +9,8 @@ module Cnvrg
|
|
|
9
9
|
class Datafiles
|
|
10
10
|
ParallelThreads ||= Cnvrg::Helpers.parallel_threads
|
|
11
11
|
|
|
12
|
-
LARGE_FILE=1024*1024*5
|
|
13
|
-
MULTIPART_SPLIT=10000000
|
|
12
|
+
LARGE_FILE = 1024 * 1024 * 5
|
|
13
|
+
MULTIPART_SPLIT = 10000000
|
|
14
14
|
|
|
15
15
|
attr_reader :base_resource
|
|
16
16
|
|
|
@@ -20,11 +20,20 @@ module Cnvrg
|
|
|
20
20
|
@dataset = dataset
|
|
21
21
|
@base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
|
|
22
22
|
@downloader = @dataset.get_storage_client
|
|
23
|
+
@token_issue_time = Time.current
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def refresh_storage_token
|
|
27
|
+
current_time = Time.current
|
|
28
|
+
if current_time - @token_issue_time > 3.hours
|
|
29
|
+
@downloader = @dataset.get_storage_client
|
|
30
|
+
@token_issue_time = Time.current
|
|
31
|
+
end
|
|
23
32
|
end
|
|
24
33
|
|
|
25
34
|
def check_file_sha1(filename, org_sha1, tag: 'conflict')
|
|
26
35
|
file_loc = "#{Dir.pwd}/#{filename}"
|
|
27
|
-
sha1 =
|
|
36
|
+
sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
|
|
28
37
|
return 0 if sha1 == org_sha1
|
|
29
38
|
FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
|
|
30
39
|
return 1
|
|
@@ -43,7 +52,7 @@ module Cnvrg
|
|
|
43
52
|
end
|
|
44
53
|
raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
|
|
45
54
|
end
|
|
46
|
-
paths.map{|p| p.gsub(/^\.\//, '')}
|
|
55
|
+
paths.map { |p| p.gsub(/^\.\//, '') }
|
|
47
56
|
end
|
|
48
57
|
|
|
49
58
|
def check_files_sha1(files, resolver, tag)
|
|
@@ -74,11 +83,11 @@ module Cnvrg
|
|
|
74
83
|
prefix = @dataset.get_dataset["bucket_prefix"]
|
|
75
84
|
batch_size = 10000
|
|
76
85
|
pbar = ProgressBar.create(:title => "Download Progress",
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
86
|
+
:progress_mark => '=',
|
|
87
|
+
:format => "%b%i| %c Files downloaded",
|
|
88
|
+
:starting_at => 0,
|
|
89
|
+
:total => nil,
|
|
90
|
+
:autofinish => true)
|
|
82
91
|
parallel_options = {
|
|
83
92
|
in_threads: ParallelThreads,
|
|
84
93
|
in_processes: Cnvrg::CLI::ParallelProcesses,
|
|
@@ -107,7 +116,7 @@ module Cnvrg
|
|
|
107
116
|
if response.present?
|
|
108
117
|
msg = response['result']
|
|
109
118
|
else
|
|
110
|
-
msg =
|
|
119
|
+
msg = "Cant save changes in the dataset"
|
|
111
120
|
end
|
|
112
121
|
|
|
113
122
|
Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
|
|
@@ -117,6 +126,7 @@ module Cnvrg
|
|
|
117
126
|
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
|
|
118
127
|
begin
|
|
119
128
|
Cnvrg::Logger.log_info("Sending Upload Files request")
|
|
129
|
+
refresh_storage_token
|
|
120
130
|
error = nil
|
|
121
131
|
upload_resp = nil
|
|
122
132
|
10.times do
|
|
@@ -161,9 +171,9 @@ module Cnvrg
|
|
|
161
171
|
end
|
|
162
172
|
@temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
|
|
163
173
|
end
|
|
164
|
-
blob_ids = files.values.map {|f| f['bv_id']}
|
|
174
|
+
blob_ids = files.values.map { |f| f['bv_id'] }
|
|
165
175
|
if blob_ids.present?
|
|
166
|
-
dirs = tree.keys.select {|k| tree[k].nil?} || []
|
|
176
|
+
dirs = tree.keys.select { |k| tree[k].nil? } || []
|
|
167
177
|
Cnvrg::Logger.info("Sending Upload files save")
|
|
168
178
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
|
|
169
179
|
unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
|
@@ -198,12 +208,12 @@ module Cnvrg
|
|
|
198
208
|
file_size = File.size(absolute_path).to_f
|
|
199
209
|
mime_type = MimeMagic.by_path(absolute_path)
|
|
200
210
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
|
201
|
-
sha1 =
|
|
211
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
|
202
212
|
if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
|
|
203
213
|
is_valid = false
|
|
204
214
|
begin
|
|
205
|
-
|
|
206
|
-
|
|
215
|
+
content = open(absolute_path).read()
|
|
216
|
+
hash = YAML.load(open(absolute_path).read())
|
|
207
217
|
# if level 1 keys count is 1
|
|
208
218
|
if hash.keys.count == 1
|
|
209
219
|
if hash["tags"].present?
|
|
@@ -211,7 +221,7 @@ module Cnvrg
|
|
|
211
221
|
elsif hash[hash.keys.first].class != Hash
|
|
212
222
|
is_valid = true
|
|
213
223
|
end
|
|
214
|
-
|
|
224
|
+
# if level 1 keys count is greater than 1
|
|
215
225
|
elsif hash.keys.count > 1
|
|
216
226
|
if hash["tags"].present? and hash["tags"].class == Hash
|
|
217
227
|
is_valid = false
|
|
@@ -245,6 +255,7 @@ module Cnvrg
|
|
|
245
255
|
return s3_res
|
|
246
256
|
end
|
|
247
257
|
end
|
|
258
|
+
|
|
248
259
|
def upload_tar_file(absolute_path, relative_path, commit_sha1)
|
|
249
260
|
begin
|
|
250
261
|
file_name = File.basename relative_path
|
|
@@ -252,24 +263,24 @@ module Cnvrg
|
|
|
252
263
|
mime_type = MimeMagic.by_path(absolute_path)
|
|
253
264
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
|
254
265
|
begin
|
|
255
|
-
chunked_bytes = [100, (file_size*0.01)].min
|
|
266
|
+
chunked_bytes = [100, (file_size * 0.01)].min
|
|
256
267
|
total_yanked = ""
|
|
257
268
|
open(absolute_path, "rb") do |f|
|
|
258
269
|
total_yanked = f.read(chunked_bytes)
|
|
259
270
|
end
|
|
260
271
|
if !total_yanked.empty?
|
|
261
|
-
sha1 =
|
|
272
|
+
sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
|
|
262
273
|
else
|
|
263
|
-
sha1 =
|
|
274
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
|
264
275
|
end
|
|
265
276
|
rescue
|
|
266
|
-
sha1 =
|
|
277
|
+
sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
|
|
267
278
|
end
|
|
268
279
|
|
|
269
280
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
|
270
281
|
commit_sha1: commit_sha1, file_name: file_name,
|
|
271
282
|
file_size: file_size, file_content_type: content_type, sha1: sha1,
|
|
272
|
-
|
|
283
|
+
new_version: true})
|
|
273
284
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
|
274
285
|
path = upload_resp["result"]["path"]
|
|
275
286
|
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
|
@@ -305,14 +316,15 @@ module Cnvrg
|
|
|
305
316
|
return false
|
|
306
317
|
|
|
307
318
|
end
|
|
308
|
-
|
|
319
|
+
|
|
320
|
+
def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
|
|
309
321
|
file_name = File.basename relative_path
|
|
310
322
|
file_size = File.size(absolute_path).to_f
|
|
311
323
|
content_type = "text/x-log"
|
|
312
324
|
upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
data_commit_sha1:data_commit_sha})
|
|
325
|
+
file_name: file_name, log_date: Time.now,
|
|
326
|
+
file_size: file_size, file_content_type: content_type,
|
|
327
|
+
data_commit_sha1: data_commit_sha})
|
|
316
328
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
|
317
329
|
path = upload_resp["result"]["path"]
|
|
318
330
|
s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
|
|
@@ -403,15 +415,15 @@ module Cnvrg
|
|
|
403
415
|
|
|
404
416
|
end
|
|
405
417
|
|
|
406
|
-
def upload_cnvrg_image(absolute_path, image_name,secret)
|
|
418
|
+
def upload_cnvrg_image(absolute_path, image_name, secret)
|
|
407
419
|
file_name = File.basename relative_path
|
|
408
420
|
file_size = File.size(absolute_path).to_f
|
|
409
421
|
mime_type = MimeMagic.by_path(absolute_path)
|
|
410
422
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
|
411
423
|
upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
424
|
+
file_name: file_name,
|
|
425
|
+
file_size: file_size, file_content_type: content_type,
|
|
426
|
+
secret: secret})
|
|
415
427
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
|
416
428
|
path = upload_resp["result"]["path"]
|
|
417
429
|
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
|
@@ -455,7 +467,7 @@ module Cnvrg
|
|
|
455
467
|
sts_path = upload_resp["result"]["path_sts"]
|
|
456
468
|
|
|
457
469
|
retries = 0
|
|
458
|
-
success= false
|
|
470
|
+
success = false
|
|
459
471
|
while !success and retries < 20
|
|
460
472
|
begin
|
|
461
473
|
if !Helpers.is_verify_ssl
|
|
@@ -465,7 +477,7 @@ module Cnvrg
|
|
|
465
477
|
end
|
|
466
478
|
success = true
|
|
467
479
|
rescue => e
|
|
468
|
-
retries +=1
|
|
480
|
+
retries += 1
|
|
469
481
|
sleep(5)
|
|
470
482
|
|
|
471
483
|
end
|
|
@@ -477,33 +489,33 @@ module Cnvrg
|
|
|
477
489
|
key = split[0]
|
|
478
490
|
iv = split[1]
|
|
479
491
|
is_s3 = upload_resp["result"]["is_s3"]
|
|
480
|
-
access =
|
|
492
|
+
access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
|
|
481
493
|
|
|
482
|
-
secret =
|
|
494
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
|
|
483
495
|
|
|
484
|
-
session =
|
|
485
|
-
region =
|
|
496
|
+
session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
|
|
497
|
+
region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
|
|
486
498
|
|
|
487
|
-
bucket =
|
|
488
|
-
server_side_encryption =upload_resp["result"]["server_side_encryption"]
|
|
499
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
|
|
500
|
+
server_side_encryption = upload_resp["result"]["server_side_encryption"]
|
|
489
501
|
use_accelerate_endpoint = false
|
|
490
502
|
if is_s3 or is_s3.nil?
|
|
491
503
|
|
|
492
|
-
use_accelerate_endpoint =true
|
|
504
|
+
use_accelerate_endpoint = true
|
|
493
505
|
client = Aws::S3::Client.new(
|
|
494
|
-
:access_key_id =>access,
|
|
506
|
+
:access_key_id => access,
|
|
495
507
|
:secret_access_key => secret,
|
|
496
508
|
:session_token => session,
|
|
497
509
|
:region => region,
|
|
498
510
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
499
511
|
else
|
|
500
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
|
|
512
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
|
|
501
513
|
use_accelerate_endpoint = false
|
|
502
514
|
client = Aws::S3::Client.new(
|
|
503
|
-
:access_key_id =>access,
|
|
515
|
+
:access_key_id => access,
|
|
504
516
|
:secret_access_key => secret,
|
|
505
517
|
:region => region,
|
|
506
|
-
:endpoint=> endpoint
|
|
518
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
|
507
519
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
508
520
|
end
|
|
509
521
|
|
|
@@ -515,13 +527,11 @@ module Cnvrg
|
|
|
515
527
|
end
|
|
516
528
|
|
|
517
529
|
|
|
518
|
-
|
|
519
|
-
|
|
520
530
|
s3 = Aws::S3::Resource.new(client: client)
|
|
521
531
|
|
|
522
532
|
resp = s3.bucket(bucket).
|
|
523
|
-
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
|
524
|
-
upload_file(file_path,options)
|
|
533
|
+
object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
|
|
534
|
+
upload_file(file_path, options)
|
|
525
535
|
|
|
526
536
|
|
|
527
537
|
return resp
|
|
@@ -578,6 +588,7 @@ module Cnvrg
|
|
|
578
588
|
response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
|
579
589
|
return Cnvrg::CLI.is_response_success(response, false)
|
|
580
590
|
end
|
|
591
|
+
|
|
581
592
|
def download_list_files_in_query(response, dataset_home)
|
|
582
593
|
sts_path = response["path_sts"]
|
|
583
594
|
if !Helpers.is_verify_ssl
|
|
@@ -589,29 +600,29 @@ module Cnvrg
|
|
|
589
600
|
key = split[0]
|
|
590
601
|
iv = split[1]
|
|
591
602
|
|
|
592
|
-
access =
|
|
603
|
+
access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
|
|
593
604
|
|
|
594
|
-
secret =
|
|
605
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
|
|
595
606
|
|
|
596
|
-
session =
|
|
597
|
-
region =
|
|
607
|
+
session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
|
|
608
|
+
region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
|
|
598
609
|
|
|
599
|
-
bucket =
|
|
610
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
|
|
600
611
|
is_s3 = response["is_s3"]
|
|
601
612
|
if is_s3 or is_s3.nil?
|
|
602
613
|
client = Aws::S3::Client.new(
|
|
603
|
-
:access_key_id =>access,
|
|
614
|
+
:access_key_id => access,
|
|
604
615
|
:secret_access_key => secret,
|
|
605
616
|
:session_token => session,
|
|
606
617
|
:region => region,
|
|
607
618
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
608
619
|
else
|
|
609
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
|
|
620
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
|
|
610
621
|
client = Aws::S3::Client.new(
|
|
611
|
-
:access_key_id =>access,
|
|
622
|
+
:access_key_id => access,
|
|
612
623
|
:secret_access_key => secret,
|
|
613
624
|
:region => region,
|
|
614
|
-
:endpoint=> endpoint
|
|
625
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
|
615
626
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
616
627
|
end
|
|
617
628
|
list_files = response["files"]
|
|
@@ -628,18 +639,18 @@ module Cnvrg
|
|
|
628
639
|
}
|
|
629
640
|
download_count = 0
|
|
630
641
|
Parallel.map((list_files), parallel_options) do |f|
|
|
631
|
-
file_key =
|
|
632
|
-
begin
|
|
642
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
|
|
633
643
|
begin
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
644
|
+
begin
|
|
645
|
+
dir = File.dirname f["fullpath"]
|
|
646
|
+
FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
|
|
647
|
+
end
|
|
637
648
|
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
649
|
+
File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
|
|
650
|
+
resp = client.get_object({bucket: bucket,
|
|
651
|
+
key: file_key}, target: file)
|
|
652
|
+
end
|
|
653
|
+
download_count += 1
|
|
643
654
|
rescue
|
|
644
655
|
end
|
|
645
656
|
|
|
@@ -651,97 +662,95 @@ module Cnvrg
|
|
|
651
662
|
end
|
|
652
663
|
|
|
653
664
|
|
|
654
|
-
|
|
655
|
-
|
|
656
665
|
end
|
|
657
666
|
|
|
658
667
|
|
|
659
|
-
def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
|
|
668
|
+
def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
|
|
660
669
|
begin
|
|
661
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1
|
|
670
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
|
|
662
671
|
Cnvrg::CLI.is_response_success(res, false)
|
|
663
|
-
|
|
664
|
-
|
|
672
|
+
if res["result"]
|
|
673
|
+
file_url = res["result"]["file_url"]
|
|
665
674
|
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
absolute_path += ".conflict" if conflict
|
|
687
|
-
sts_path = download_resp["result"]["path_sts"]
|
|
688
|
-
retries = 0
|
|
689
|
-
success= false
|
|
690
|
-
while !success and retries < 20
|
|
691
|
-
begin
|
|
692
|
-
if !Helpers.is_verify_ssl
|
|
693
|
-
body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
|
|
694
|
-
else
|
|
695
|
-
body = open(sts_path).read
|
|
696
|
-
end
|
|
697
|
-
success = true
|
|
698
|
-
rescue => e
|
|
699
|
-
retries +=1
|
|
700
|
-
sleep(5)
|
|
675
|
+
if as_link
|
|
676
|
+
return res["result"]
|
|
677
|
+
end
|
|
678
|
+
# begin
|
|
679
|
+
# if !Helpers.is_verify_ssl
|
|
680
|
+
# tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
|
|
681
|
+
#
|
|
682
|
+
# else
|
|
683
|
+
# tempfile = Down.download(file_url,open_timeout: 60)
|
|
684
|
+
#
|
|
685
|
+
# end
|
|
686
|
+
#
|
|
687
|
+
# FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
|
|
688
|
+
# return true
|
|
689
|
+
# rescue
|
|
690
|
+
#
|
|
691
|
+
# end
|
|
692
|
+
download_resp = res
|
|
693
|
+
filename = download_resp["result"]["filename"]
|
|
701
694
|
|
|
695
|
+
absolute_path += ".conflict" if conflict
|
|
696
|
+
sts_path = download_resp["result"]["path_sts"]
|
|
697
|
+
retries = 0
|
|
698
|
+
success = false
|
|
699
|
+
while !success and retries < 20
|
|
700
|
+
begin
|
|
701
|
+
if !Helpers.is_verify_ssl
|
|
702
|
+
body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
|
|
703
|
+
else
|
|
704
|
+
body = open(sts_path).read
|
|
702
705
|
end
|
|
706
|
+
success = true
|
|
707
|
+
rescue => e
|
|
708
|
+
retries += 1
|
|
709
|
+
sleep(5)
|
|
710
|
+
|
|
703
711
|
end
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
712
|
+
end
|
|
713
|
+
if !success
|
|
714
|
+
return false
|
|
715
|
+
end
|
|
716
|
+
split = body.split("\n")
|
|
717
|
+
key = split[0]
|
|
718
|
+
iv = split[1]
|
|
710
719
|
|
|
711
|
-
|
|
720
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
|
712
721
|
|
|
713
|
-
|
|
722
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
|
|
714
723
|
|
|
715
|
-
|
|
716
|
-
|
|
724
|
+
session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
|
|
725
|
+
region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
|
|
717
726
|
|
|
718
|
-
|
|
719
|
-
|
|
727
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
|
|
728
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
|
|
720
729
|
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
730
|
+
is_s3 = download_resp["result"]["is_s3"]
|
|
731
|
+
if is_s3 or is_s3.nil?
|
|
732
|
+
client = Aws::S3::Client.new(
|
|
733
|
+
:access_key_id => access,
|
|
734
|
+
:secret_access_key => secret,
|
|
735
|
+
:session_token => session,
|
|
736
|
+
:region => region,
|
|
737
|
+
:http_open_timeout => 60, :retry_limit => 20)
|
|
738
|
+
else
|
|
739
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
|
|
740
|
+
client = Aws::S3::Client.new(
|
|
741
|
+
:access_key_id => access,
|
|
742
|
+
:secret_access_key => secret,
|
|
743
|
+
:region => region,
|
|
744
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
|
745
|
+
:http_open_timeout => 60, :retry_limit => 20)
|
|
746
|
+
end
|
|
738
747
|
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
end
|
|
743
|
-
return true
|
|
748
|
+
File.open(project_home + "/" + absolute_path, 'w+') do |file|
|
|
749
|
+
resp = client.get_object({bucket: bucket,
|
|
750
|
+
key: file_key}, target: file)
|
|
744
751
|
end
|
|
752
|
+
return true
|
|
753
|
+
end
|
|
745
754
|
|
|
746
755
|
rescue => e
|
|
747
756
|
return false
|
|
@@ -751,7 +760,7 @@ module Cnvrg
|
|
|
751
760
|
|
|
752
761
|
def download_data_file(commit_sha1, dataset_home)
|
|
753
762
|
begin
|
|
754
|
-
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
|
|
763
|
+
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
|
|
755
764
|
Cnvrg::CLI.is_response_success(res, false)
|
|
756
765
|
if res["result"]
|
|
757
766
|
download_resp = res
|
|
@@ -768,38 +777,38 @@ module Cnvrg
|
|
|
768
777
|
iv = split[1]
|
|
769
778
|
|
|
770
779
|
|
|
771
|
-
access =
|
|
780
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
|
772
781
|
|
|
773
|
-
secret =
|
|
782
|
+
secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
|
|
774
783
|
|
|
775
|
-
session =
|
|
776
|
-
region =
|
|
784
|
+
session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
|
|
785
|
+
region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
|
|
777
786
|
|
|
778
|
-
bucket =
|
|
779
|
-
file_key =
|
|
787
|
+
bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
|
|
788
|
+
file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
|
|
780
789
|
|
|
781
790
|
is_s3 = download_resp["result"]["is_s3"]
|
|
782
791
|
if is_s3 or is_s3.nil?
|
|
783
792
|
client = Aws::S3::Client.new(
|
|
784
|
-
:access_key_id =>access,
|
|
793
|
+
:access_key_id => access,
|
|
785
794
|
:secret_access_key => secret,
|
|
786
795
|
:session_token => session,
|
|
787
796
|
:region => region,
|
|
788
797
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
789
798
|
else
|
|
790
|
-
endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
|
|
799
|
+
endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
|
|
791
800
|
client = Aws::S3::Client.new(
|
|
792
|
-
:access_key_id =>access,
|
|
801
|
+
:access_key_id => access,
|
|
793
802
|
:secret_access_key => secret,
|
|
794
803
|
:region => region,
|
|
795
|
-
:endpoint=> endpoint
|
|
804
|
+
:endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
|
|
796
805
|
:http_open_timeout => 60, :retry_limit => 20)
|
|
797
806
|
end
|
|
798
807
|
|
|
799
808
|
|
|
800
|
-
File.open(dataset_home+"/"+filename, 'w+') do |file|
|
|
809
|
+
File.open(dataset_home + "/" + filename, 'w+') do |file|
|
|
801
810
|
resp = client.get_object({bucket: bucket,
|
|
802
|
-
|
|
811
|
+
key: file_key}, target: file)
|
|
803
812
|
end
|
|
804
813
|
return filename
|
|
805
814
|
end
|
|
@@ -810,7 +819,7 @@ module Cnvrg
|
|
|
810
819
|
end
|
|
811
820
|
end
|
|
812
821
|
|
|
813
|
-
def download_file(absolute_path, relative_path, project_home, conflict=false)
|
|
822
|
+
def download_file(absolute_path, relative_path, project_home, conflict = false)
|
|
814
823
|
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
|
815
824
|
Cnvrg::CLI.is_response_success(res, false)
|
|
816
825
|
if res["result"]
|
|
@@ -848,7 +857,7 @@ module Cnvrg
|
|
|
848
857
|
puts FileUtils.rmtree("#{absolute_path}")
|
|
849
858
|
end
|
|
850
859
|
|
|
851
|
-
def revoke_download_file(absolute_path, filename, conflict=false)
|
|
860
|
+
def revoke_download_file(absolute_path, filename, conflict = false)
|
|
852
861
|
begin
|
|
853
862
|
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
|
854
863
|
|
|
@@ -859,6 +868,7 @@ module Cnvrg
|
|
|
859
868
|
return false
|
|
860
869
|
end
|
|
861
870
|
end
|
|
871
|
+
|
|
862
872
|
def revoke_download(tar_files, extracted_files)
|
|
863
873
|
begin
|
|
864
874
|
|
|
@@ -872,32 +882,35 @@ module Cnvrg
|
|
|
872
882
|
return true
|
|
873
883
|
|
|
874
884
|
end
|
|
885
|
+
|
|
875
886
|
def delete_commit(commit_sha1)
|
|
876
887
|
response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
|
|
877
888
|
Cnvrg::CLI.is_response_success(response, true)
|
|
878
889
|
return response
|
|
879
890
|
end
|
|
891
|
+
|
|
880
892
|
def get_commit(commit_sha1)
|
|
881
893
|
response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
|
|
882
894
|
Cnvrg::CLI.is_response_success(response, true)
|
|
883
895
|
return response
|
|
884
896
|
end
|
|
885
897
|
|
|
886
|
-
def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
|
|
898
|
+
def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
|
|
887
899
|
begin
|
|
888
900
|
#if we are pushing with force or to branch we dont need to send current/next commit cause we want to
|
|
889
901
|
# create a new commit.
|
|
890
902
|
idx = (force || new_branch) ? {} : dataset.get_idx
|
|
891
903
|
commit = idx[:commit]
|
|
892
904
|
next_commit = idx[:next_commit]
|
|
893
|
-
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
|
|
894
|
-
username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
|
|
905
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
|
|
906
|
+
username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
|
|
895
907
|
Cnvrg::CLI.is_response_success(response, true)
|
|
896
908
|
return response
|
|
897
909
|
rescue => e
|
|
898
910
|
return false
|
|
899
911
|
end
|
|
900
912
|
end
|
|
913
|
+
|
|
901
914
|
def last_valid_commit()
|
|
902
915
|
begin
|
|
903
916
|
#if we are pushing with force or to branch we dont need to send current/next commit cause we want to
|
|
@@ -910,9 +923,9 @@ module Cnvrg
|
|
|
910
923
|
end
|
|
911
924
|
end
|
|
912
925
|
|
|
913
|
-
def end_commit(commit_sha1,force, success: true, uploaded_files: 0
|
|
926
|
+
def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
|
|
914
927
|
begin
|
|
915
|
-
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
|
|
928
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
|
|
916
929
|
Cnvrg::CLI.is_response_success(response, true)
|
|
917
930
|
return response
|
|
918
931
|
rescue => e
|
|
@@ -938,13 +951,13 @@ module Cnvrg
|
|
|
938
951
|
end
|
|
939
952
|
|
|
940
953
|
def get_trees(commit: "latest")
|
|
941
|
-
response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
|
|
954
|
+
response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
|
|
942
955
|
return nil unless Cnvrg::CLI.is_response_success(response, false)
|
|
943
956
|
response['result']['files']
|
|
944
957
|
end
|
|
945
958
|
|
|
946
959
|
def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
|
|
947
|
-
response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
|
|
960
|
+
response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
|
|
948
961
|
unless Cnvrg::CLI.is_response_success(response, false)
|
|
949
962
|
Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
|
|
950
963
|
return nil
|
|
@@ -975,26 +988,26 @@ module Cnvrg
|
|
|
975
988
|
end
|
|
976
989
|
end
|
|
977
990
|
|
|
978
|
-
def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
|
|
991
|
+
def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
|
|
979
992
|
begin
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
993
|
+
last_chunk_size = chunk_size
|
|
994
|
+
q = {commit: commit, chunk_size: chunk_size}
|
|
995
|
+
overall = 0
|
|
996
|
+
while last_chunk_size > 0
|
|
997
|
+
response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
|
|
998
|
+
if Cnvrg::CLI.is_response_success(response, false)
|
|
999
|
+
files = response['files']
|
|
1000
|
+
data_home = "#{Dir.pwd}/#{response['name']}"
|
|
1001
|
+
last_chunk_size = files['keys'].length
|
|
1002
|
+
break if last_chunk_size == 0
|
|
1003
|
+
res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
|
|
1004
|
+
overall += last_chunk_size
|
|
1005
|
+
q[:latest] = files['latest']
|
|
1006
|
+
else
|
|
1007
|
+
last_chunk_size = 0
|
|
1008
|
+
end
|
|
995
1009
|
end
|
|
996
|
-
|
|
997
|
-
Cnvrg::Result.new(true, "Cloned #{overall} files!")
|
|
1010
|
+
Cnvrg::Result.new(true, "Cloned #{overall} files!")
|
|
998
1011
|
rescue => e
|
|
999
1012
|
Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
|
|
1000
1013
|
|
|
@@ -1002,42 +1015,43 @@ module Cnvrg
|
|
|
1002
1015
|
end
|
|
1003
1016
|
|
|
1004
1017
|
def generate_parallel_idx
|
|
1005
|
-
|
|
1018
|
+
|
|
1006
1019
|
end
|
|
1007
1020
|
|
|
1008
|
-
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
|
|
1021
|
+
def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
|
|
1009
1022
|
begin
|
|
1023
|
+
refresh_storage_token
|
|
1010
1024
|
parallel_options = {
|
|
1011
1025
|
in_threads: ParallelThreads,
|
|
1012
1026
|
isolation: true
|
|
1013
1027
|
}
|
|
1014
1028
|
Parallel.map(files["keys"], parallel_options) do |f|
|
|
1015
1029
|
begin
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1030
|
+
local_path = @dataset.local_path + '/' + f['name']
|
|
1031
|
+
Cnvrg::Logger.log_info("Downloading #{local_path}")
|
|
1032
|
+
progressbar.progress += 1 if progressbar.present?
|
|
1033
|
+
if local_path.end_with? "/"
|
|
1034
|
+
@downloader.mkdir(local_path, recursive: true)
|
|
1035
|
+
next
|
|
1036
|
+
end
|
|
1023
1037
|
# blob
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1038
|
+
local_path = "#{local_path}.conflict" if conflict
|
|
1039
|
+
storage_path = f["path"]
|
|
1040
|
+
if File.exists? local_path
|
|
1041
|
+
Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
|
|
1042
|
+
next
|
|
1043
|
+
end
|
|
1044
|
+
resp = @downloader.download(storage_path, local_path)
|
|
1045
|
+
Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
|
|
1032
1046
|
rescue => e
|
|
1033
1047
|
Cnvrg::Logger.log_error(e)
|
|
1034
1048
|
end
|
|
1035
1049
|
end
|
|
1036
|
-
return Cnvrg::Result.new(true,"Downloaded successfully")
|
|
1050
|
+
return Cnvrg::Result.new(true, "Downloaded successfully")
|
|
1037
1051
|
rescue => e
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
end
|
|
1052
|
+
Cnvrg::Logger.log_error(e)
|
|
1053
|
+
return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
|
|
1041
1054
|
end
|
|
1055
|
+
end
|
|
1042
1056
|
end
|
|
1043
1057
|
end
|
data/lib/cnvrg/experiment.rb
CHANGED
|
@@ -108,7 +108,7 @@ module Cnvrg
|
|
|
108
108
|
def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
|
|
109
109
|
sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
|
|
110
110
|
git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
|
|
111
|
-
email_notification_error: false, email_notification_success: false)
|
|
111
|
+
email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
|
|
112
112
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
|
113
113
|
commit_sha1: commit_to_run,
|
|
114
114
|
instance_type: instance_type,
|
|
@@ -124,7 +124,8 @@ module Cnvrg
|
|
|
124
124
|
restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
|
|
125
125
|
prerun: prerun, requirements: requirements, recurring: recurring,
|
|
126
126
|
email_notification_error: email_notification_error,
|
|
127
|
-
email_notification_success: email_notification_success
|
|
127
|
+
email_notification_success: email_notification_success,
|
|
128
|
+
emails_to_notify: emails_to_notify})
|
|
128
129
|
|
|
129
130
|
return response
|
|
130
131
|
end
|
data/lib/cnvrg/files.rb
CHANGED
|
@@ -7,7 +7,7 @@ require 'cnvrg/result'
|
|
|
7
7
|
module Cnvrg
|
|
8
8
|
class Files
|
|
9
9
|
ParallelThreads = Cnvrg::Helpers.parallel_threads
|
|
10
|
-
VALID_FILE_NAME = /[\x00
|
|
10
|
+
VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
|
|
11
11
|
LARGE_FILE=1024*1024*5
|
|
12
12
|
MULTIPART_SPLIT=10000000
|
|
13
13
|
|
|
@@ -27,8 +27,16 @@ module Cnvrg
|
|
|
27
27
|
@custom_progess = false
|
|
28
28
|
@cli = cli
|
|
29
29
|
@options = options
|
|
30
|
+
@token_issue_time = Time.current
|
|
30
31
|
end
|
|
31
32
|
|
|
33
|
+
def refresh_storage_token
|
|
34
|
+
current_time = Time.current
|
|
35
|
+
if current_time - @token_issue_time > 3.hours
|
|
36
|
+
@client = @project.get_storage_client
|
|
37
|
+
@token_issue_time = Time.current
|
|
38
|
+
end
|
|
39
|
+
end
|
|
32
40
|
|
|
33
41
|
def self.valid_file_name?(fullpath)
|
|
34
42
|
VALID_FILE_NAME.match(fullpath).blank?
|
|
@@ -97,7 +105,13 @@ module Cnvrg
|
|
|
97
105
|
files = res['files']
|
|
98
106
|
|
|
99
107
|
#upload files
|
|
108
|
+
token_mutex = Mutex.new
|
|
100
109
|
blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
|
|
110
|
+
|
|
111
|
+
token_mutex.synchronize {
|
|
112
|
+
refresh_storage_token
|
|
113
|
+
}
|
|
114
|
+
|
|
101
115
|
begin
|
|
102
116
|
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
|
|
103
117
|
rescue => e
|
|
@@ -721,8 +735,15 @@ module Cnvrg
|
|
|
721
735
|
in_threads: Cnvrg::Helpers.parallel_threads,
|
|
722
736
|
isolation: true
|
|
723
737
|
}
|
|
738
|
+
|
|
739
|
+
token_mutex = Mutex.new
|
|
740
|
+
|
|
724
741
|
Parallel.map(files["keys"], parallel_options) do |f|
|
|
725
742
|
|
|
743
|
+
token_mutex.synchronize {
|
|
744
|
+
refresh_storage_token
|
|
745
|
+
}
|
|
746
|
+
|
|
726
747
|
file_path = f["name"]
|
|
727
748
|
if file_path.end_with? "/"
|
|
728
749
|
# dir
|
data/lib/cnvrg/project.rb
CHANGED
|
@@ -370,6 +370,13 @@ module Cnvrg
|
|
|
370
370
|
File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
|
|
371
371
|
end
|
|
372
372
|
|
|
373
|
+
def generate_git_diff
|
|
374
|
+
git_diff = `git diff --name-only`
|
|
375
|
+
git_diff.split("\n")
|
|
376
|
+
rescue
|
|
377
|
+
[]
|
|
378
|
+
end
|
|
379
|
+
|
|
373
380
|
def generate_output_dir(output_dir)
|
|
374
381
|
Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
|
|
375
382
|
upload_list = []
|
|
@@ -387,7 +394,7 @@ module Cnvrg
|
|
|
387
394
|
upload_list << output_dir + "/"
|
|
388
395
|
end
|
|
389
396
|
Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
|
|
390
|
-
|
|
397
|
+
upload_list
|
|
391
398
|
end
|
|
392
399
|
def generate_output_dir_tmp(output_dir)
|
|
393
400
|
upload_list = []
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: cnvrg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.6.
|
|
4
|
+
version: 1.6.38
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Yochay Ettun
|
|
@@ -10,7 +10,7 @@ authors:
|
|
|
10
10
|
autorequire:
|
|
11
11
|
bindir: bin
|
|
12
12
|
cert_chain: []
|
|
13
|
-
date: 2020-
|
|
13
|
+
date: 2020-06-23 00:00:00.000000000 Z
|
|
14
14
|
dependencies:
|
|
15
15
|
- !ruby/object:Gem::Dependency
|
|
16
16
|
name: bundler
|