cnvrg 1.6.31 → 1.6.38

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d7dee30d3b567e7e1ab181aed63426524d6f04459fe365ce5d60cc4c7671ed03
4
- data.tar.gz: f8fcb9901d5bc06471500246a63f3ff28d5593fe6564f8cf0930d6d41342ed16
3
+ metadata.gz: e5953510af9633c925be99ac1463aef31667eed5f83f5b0c610672ab9fe375cb
4
+ data.tar.gz: 7bd8fce2d93a20ed695e4c202f2b29092de84bf90332385a3921844046a0c591
5
5
  SHA512:
6
- metadata.gz: 935d27a9023559e57694c726515846175bcd892a1906fdf058983268ba2e2b81ecc41bf8b31a689b01302cdc8afaef8dd6d5130d686857d6cfe30e4fdec3f45a
7
- data.tar.gz: 9ebd4da0e91c62e64712453eab68c5fe7914dfc6ec5ad26aae9f6fbbf6f1cdfce8d187f7d8f8a1c0aa5f1955c6842ae317da9fca2238d20c8410c3d6c31158a3
6
+ metadata.gz: 8b93cd899060800095895967dbe2b8f3391e2d77b6f29ea05cd6651de4d26de2ba24fdaedaa8dafad82a22e250d95abf7651d2b77afe2dcf68b883461774c965
7
+ data.tar.gz: a4dc324e6f58b628013bb00278a74233f6b2c6e4ffa260e9edf74ea2557ca144907449078defb1abdb22ffe411fdde196b2e029bd66e67233140cf54d6caf67d
@@ -1184,7 +1184,7 @@ module Cnvrg
1184
1184
  end
1185
1185
 
1186
1186
  desc '', '', :hide => true
1187
- def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
1187
+ def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
1188
1188
  begin
1189
1189
  verify_logged_in(false)
1190
1190
  log_start(__method__, args, options)
@@ -1204,7 +1204,7 @@ module Cnvrg
1204
1204
  log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
1205
1205
  number_of_chunks = (@files.size.to_f / chunk_size).ceil
1206
1206
  if commit.blank?
1207
- response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
1207
+ response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
1208
1208
  unless response #means we failed in the start commit.
1209
1209
  raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
1210
1210
  end
@@ -1837,8 +1837,6 @@ module Cnvrg
1837
1837
  end
1838
1838
  clone_resp = Project.clone_dir(slug, owner, project_name,git)
1839
1839
  project_home = Dir.pwd + "/" + project_name
1840
-
1841
-
1842
1840
  end
1843
1841
 
1844
1842
  if clone_resp
@@ -2213,6 +2211,7 @@ module Cnvrg
2213
2211
  method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
2214
2212
  method_option :files, :type => :string, :aliases => ["--files"], :default => nil
2215
2213
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2214
+ method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
2216
2215
  method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
2217
2216
  method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
2218
2217
 
@@ -2237,19 +2236,21 @@ module Cnvrg
2237
2236
  spec_files_to_upload = spec_files_to_upload.split(",")
2238
2237
  end
2239
2238
  if @project.is_git
2239
+ list = []
2240
2240
  git_output_dir = options["output_dir"] || output_dir
2241
2241
  if git_output_dir.present?
2242
2242
  if git_output_dir.ends_with? "/"
2243
2243
  git_output_dir = git_output_dir[0..-2]
2244
2244
  end
2245
2245
  list = @project.generate_output_dir(git_output_dir)
2246
- spec_files_to_upload = list
2247
- if spec_files_to_upload.blank?
2248
- log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
2249
- return true
2250
- end
2251
- force = true
2252
2246
  end
2247
+ list += @project.generate_git_diff if options["git_diff"]
2248
+ spec_files_to_upload = list
2249
+ if spec_files_to_upload.blank?
2250
+ log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
2251
+ return true
2252
+ end
2253
+ force = true
2253
2254
  end
2254
2255
 
2255
2256
  if ignore.nil? or ignore.empty?
@@ -2894,6 +2895,7 @@ module Cnvrg
2894
2895
  method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
2895
2896
  method_option :files, :type => :string, :aliases => ["--files"], :default => nil
2896
2897
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2898
+ method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
2897
2899
  def sync(direct = true)
2898
2900
  verify_logged_in(true) if direct
2899
2901
  @project = Project.new(get_project_home)
@@ -2914,7 +2916,7 @@ module Cnvrg
2914
2916
  end
2915
2917
  invoke :upload, [false, true, direct, "",in_exp,options[:force], options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
2916
2918
  :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
2917
- :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
2919
+ :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type, :git_diff=> options["git_diff"]
2918
2920
  end
2919
2921
 
2920
2922
  desc 'run cmd', 'Runs an experiment'
@@ -2956,6 +2958,7 @@ module Cnvrg
2956
2958
  method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
2957
2959
  method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
2958
2960
  method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
2961
+ method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
2959
2962
 
2960
2963
  def run(*cmd)
2961
2964
  verify_logged_in(true)
@@ -2989,6 +2992,7 @@ module Cnvrg
2989
2992
  requirements = options["requirements"]
2990
2993
  email_notification_error = options["notify_on_error"]
2991
2994
  email_notification_success = options["notify_on_success"]
2995
+ emails = options["emails"]
2992
2996
 
2993
2997
  if !data.present? and data_query.present?
2994
2998
  log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3003,7 +3007,6 @@ module Cnvrg
3003
3007
  restart_if_stuck = options["restart_if_stuck"]
3004
3008
 
3005
3009
  options_hash = Hash[options]
3006
-
3007
3010
  if local
3008
3011
  if Cnvrg::Helpers.windows?
3009
3012
  say "Windows is currently not supported for running experiments locally"
@@ -3039,7 +3042,7 @@ module Cnvrg
3039
3042
  :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
3040
3043
  :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
3041
3044
  :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
3042
- :email_notification_error => email_notification_error, :email_notification_success => email_notification_success
3045
+ :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
3043
3046
  return
3044
3047
  end
3045
3048
 
@@ -3345,8 +3348,9 @@ module Cnvrg
3345
3348
  method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
3346
3349
  method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
3347
3350
  method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
3348
- method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
3349
- method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
3351
+ method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
3352
+ method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
3353
+ method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
3350
3354
 
3351
3355
  def exec_remote(*cmd)
3352
3356
 
@@ -3368,6 +3372,7 @@ module Cnvrg
3368
3372
  requirements = options["requirements"]
3369
3373
  email_notification_error = options["email_notification_error"]
3370
3374
  email_notification_success = options["email_notification_success"]
3375
+ emails = options["emails"]
3371
3376
  max_time = options["max_time"]
3372
3377
  if !max_time.nil? and !max_time.empty?
3373
3378
  max_time = max_time.to_i
@@ -3407,7 +3412,7 @@ module Cnvrg
3407
3412
  options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
3408
3413
  "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
3409
3414
  "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
3410
- "email_notification_error", "email_notification_success")
3415
+ "email_notification_error", "email_notification_success", "emails")
3411
3416
  exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
3412
3417
  command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
3413
3418
  commit_to_run = options["commit"] || nil
@@ -3475,7 +3480,7 @@ module Cnvrg
3475
3480
  res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
3476
3481
  periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
3477
3482
  data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
3478
- email_notification_error: email_notification_error, email_notification_success: email_notification_success)
3483
+ email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
3479
3484
  if Cnvrg::CLI.is_response_success(res)
3480
3485
  check = Helpers.checkmark()
3481
3486
  str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
@@ -186,11 +186,14 @@ module Cnvrg
186
186
  desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
187
187
  method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
188
188
  method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
189
+ method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
189
190
  def put(dataset_url, *files)
190
191
  cli = Cnvrg::CLI.new()
191
192
  dir = options[:dir]
192
193
  commit = options[:commit]
193
- cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
194
+ message = options[:message]
195
+
196
+ cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
194
197
  end
195
198
 
196
199
  desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
@@ -9,8 +9,8 @@ module Cnvrg
9
9
  class Datafiles
10
10
  ParallelThreads ||= Cnvrg::Helpers.parallel_threads
11
11
 
12
- LARGE_FILE=1024*1024*5
13
- MULTIPART_SPLIT=10000000
12
+ LARGE_FILE = 1024 * 1024 * 5
13
+ MULTIPART_SPLIT = 10000000
14
14
 
15
15
  attr_reader :base_resource
16
16
 
@@ -20,11 +20,20 @@ module Cnvrg
20
20
  @dataset = dataset
21
21
  @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
22
22
  @downloader = @dataset.get_storage_client
23
+ @token_issue_time = Time.current
24
+ end
25
+
26
+ def refresh_storage_token
27
+ current_time = Time.current
28
+ if current_time - @token_issue_time > 3.hours
29
+ @downloader = @dataset.get_storage_client
30
+ @token_issue_time = Time.current
31
+ end
23
32
  end
24
33
 
25
34
  def check_file_sha1(filename, org_sha1, tag: 'conflict')
26
35
  file_loc = "#{Dir.pwd}/#{filename}"
27
- sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
36
+ sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
28
37
  return 0 if sha1 == org_sha1
29
38
  FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
30
39
  return 1
@@ -43,7 +52,7 @@ module Cnvrg
43
52
  end
44
53
  raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
45
54
  end
46
- paths.map{|p| p.gsub(/^\.\//, '')}
55
+ paths.map { |p| p.gsub(/^\.\//, '') }
47
56
  end
48
57
 
49
58
  def check_files_sha1(files, resolver, tag)
@@ -74,11 +83,11 @@ module Cnvrg
74
83
  prefix = @dataset.get_dataset["bucket_prefix"]
75
84
  batch_size = 10000
76
85
  pbar = ProgressBar.create(:title => "Download Progress",
77
- :progress_mark => '=',
78
- :format => "%b%i| %c Files downloaded",
79
- :starting_at => 0,
80
- :total => nil,
81
- :autofinish => true)
86
+ :progress_mark => '=',
87
+ :format => "%b%i| %c Files downloaded",
88
+ :starting_at => 0,
89
+ :total => nil,
90
+ :autofinish => true)
82
91
  parallel_options = {
83
92
  in_threads: ParallelThreads,
84
93
  in_processes: Cnvrg::CLI::ParallelProcesses,
@@ -107,7 +116,7 @@ module Cnvrg
107
116
  if response.present?
108
117
  msg = response['result']
109
118
  else
110
- msg = "Cant save changes in the dataset"
119
+ msg = "Cant save changes in the dataset"
111
120
  end
112
121
 
113
122
  Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
@@ -117,6 +126,7 @@ module Cnvrg
117
126
  def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
118
127
  begin
119
128
  Cnvrg::Logger.log_info("Sending Upload Files request")
129
+ refresh_storage_token
120
130
  error = nil
121
131
  upload_resp = nil
122
132
  10.times do
@@ -161,9 +171,9 @@ module Cnvrg
161
171
  end
162
172
  @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
163
173
  end
164
- blob_ids = files.values.map {|f| f['bv_id']}
174
+ blob_ids = files.values.map { |f| f['bv_id'] }
165
175
  if blob_ids.present?
166
- dirs = tree.keys.select {|k| tree[k].nil?} || []
176
+ dirs = tree.keys.select { |k| tree[k].nil? } || []
167
177
  Cnvrg::Logger.info("Sending Upload files save")
168
178
  upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
169
179
  unless Cnvrg::CLI.is_response_success(upload_resp, false)
@@ -198,12 +208,12 @@ module Cnvrg
198
208
  file_size = File.size(absolute_path).to_f
199
209
  mime_type = MimeMagic.by_path(absolute_path)
200
210
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
201
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
211
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
202
212
  if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
203
213
  is_valid = false
204
214
  begin
205
- content = open(absolute_path).read()
206
- hash = YAML.load(open(absolute_path).read())
215
+ content = open(absolute_path).read()
216
+ hash = YAML.load(open(absolute_path).read())
207
217
  # if level 1 keys count is 1
208
218
  if hash.keys.count == 1
209
219
  if hash["tags"].present?
@@ -211,7 +221,7 @@ module Cnvrg
211
221
  elsif hash[hash.keys.first].class != Hash
212
222
  is_valid = true
213
223
  end
214
- # if level 1 keys count is greater than 1
224
+ # if level 1 keys count is greater than 1
215
225
  elsif hash.keys.count > 1
216
226
  if hash["tags"].present? and hash["tags"].class == Hash
217
227
  is_valid = false
@@ -245,6 +255,7 @@ module Cnvrg
245
255
  return s3_res
246
256
  end
247
257
  end
258
+
248
259
  def upload_tar_file(absolute_path, relative_path, commit_sha1)
249
260
  begin
250
261
  file_name = File.basename relative_path
@@ -252,24 +263,24 @@ module Cnvrg
252
263
  mime_type = MimeMagic.by_path(absolute_path)
253
264
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
254
265
  begin
255
- chunked_bytes = [100, (file_size*0.01)].min
266
+ chunked_bytes = [100, (file_size * 0.01)].min
256
267
  total_yanked = ""
257
268
  open(absolute_path, "rb") do |f|
258
269
  total_yanked = f.read(chunked_bytes)
259
270
  end
260
271
  if !total_yanked.empty?
261
- sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
272
+ sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
262
273
  else
263
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
274
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
264
275
  end
265
276
  rescue
266
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
277
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
267
278
  end
268
279
 
269
280
  upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
270
281
  commit_sha1: commit_sha1, file_name: file_name,
271
282
  file_size: file_size, file_content_type: content_type, sha1: sha1,
272
- new_version:true})
283
+ new_version: true})
273
284
  if Cnvrg::CLI.is_response_success(upload_resp, false)
274
285
  path = upload_resp["result"]["path"]
275
286
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -305,14 +316,15 @@ module Cnvrg
305
316
  return false
306
317
 
307
318
  end
308
- def upload_data_log_file(absolute_path, relative_path,data_commit_sha)
319
+
320
+ def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
309
321
  file_name = File.basename relative_path
310
322
  file_size = File.size(absolute_path).to_f
311
323
  content_type = "text/x-log"
312
324
  upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
313
- file_name: file_name, log_date: Time.now,
314
- file_size: file_size, file_content_type: content_type,
315
- data_commit_sha1:data_commit_sha})
325
+ file_name: file_name, log_date: Time.now,
326
+ file_size: file_size, file_content_type: content_type,
327
+ data_commit_sha1: data_commit_sha})
316
328
  if Cnvrg::CLI.is_response_success(upload_resp, false)
317
329
  path = upload_resp["result"]["path"]
318
330
  s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
@@ -403,15 +415,15 @@ module Cnvrg
403
415
 
404
416
  end
405
417
 
406
- def upload_cnvrg_image(absolute_path, image_name,secret)
418
+ def upload_cnvrg_image(absolute_path, image_name, secret)
407
419
  file_name = File.basename relative_path
408
420
  file_size = File.size(absolute_path).to_f
409
421
  mime_type = MimeMagic.by_path(absolute_path)
410
422
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
411
423
  upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
412
- file_name: file_name,
413
- file_size: file_size, file_content_type: content_type,
414
- secret: secret})
424
+ file_name: file_name,
425
+ file_size: file_size, file_content_type: content_type,
426
+ secret: secret})
415
427
  if Cnvrg::CLI.is_response_success(upload_resp, false)
416
428
  path = upload_resp["result"]["path"]
417
429
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -455,7 +467,7 @@ module Cnvrg
455
467
  sts_path = upload_resp["result"]["path_sts"]
456
468
 
457
469
  retries = 0
458
- success= false
470
+ success = false
459
471
  while !success and retries < 20
460
472
  begin
461
473
  if !Helpers.is_verify_ssl
@@ -465,7 +477,7 @@ module Cnvrg
465
477
  end
466
478
  success = true
467
479
  rescue => e
468
- retries +=1
480
+ retries += 1
469
481
  sleep(5)
470
482
 
471
483
  end
@@ -477,33 +489,33 @@ module Cnvrg
477
489
  key = split[0]
478
490
  iv = split[1]
479
491
  is_s3 = upload_resp["result"]["is_s3"]
480
- access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
492
+ access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
481
493
 
482
- secret = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
494
+ secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
483
495
 
484
- session = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
485
- region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
496
+ session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
497
+ region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
486
498
 
487
- bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
488
- server_side_encryption =upload_resp["result"]["server_side_encryption"]
499
+ bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
500
+ server_side_encryption = upload_resp["result"]["server_side_encryption"]
489
501
  use_accelerate_endpoint = false
490
502
  if is_s3 or is_s3.nil?
491
503
 
492
- use_accelerate_endpoint =true
504
+ use_accelerate_endpoint = true
493
505
  client = Aws::S3::Client.new(
494
- :access_key_id =>access,
506
+ :access_key_id => access,
495
507
  :secret_access_key => secret,
496
508
  :session_token => session,
497
509
  :region => region,
498
510
  :http_open_timeout => 60, :retry_limit => 20)
499
511
  else
500
- endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
512
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
501
513
  use_accelerate_endpoint = false
502
514
  client = Aws::S3::Client.new(
503
- :access_key_id =>access,
515
+ :access_key_id => access,
504
516
  :secret_access_key => secret,
505
517
  :region => region,
506
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
518
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
507
519
  :http_open_timeout => 60, :retry_limit => 20)
508
520
  end
509
521
 
@@ -515,13 +527,11 @@ module Cnvrg
515
527
  end
516
528
 
517
529
 
518
-
519
-
520
530
  s3 = Aws::S3::Resource.new(client: client)
521
531
 
522
532
  resp = s3.bucket(bucket).
523
- object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
524
- upload_file(file_path,options)
533
+ object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
534
+ upload_file(file_path, options)
525
535
 
526
536
 
527
537
  return resp
@@ -578,6 +588,7 @@ module Cnvrg
578
588
  response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
579
589
  return Cnvrg::CLI.is_response_success(response, false)
580
590
  end
591
+
581
592
  def download_list_files_in_query(response, dataset_home)
582
593
  sts_path = response["path_sts"]
583
594
  if !Helpers.is_verify_ssl
@@ -589,29 +600,29 @@ module Cnvrg
589
600
  key = split[0]
590
601
  iv = split[1]
591
602
 
592
- access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
603
+ access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
593
604
 
594
- secret = Cnvrg::Helpers.decrypt(key,iv, response["sts_s"])
605
+ secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
595
606
 
596
- session = Cnvrg::Helpers.decrypt(key,iv, response["sts_st"])
597
- region = Cnvrg::Helpers.decrypt(key,iv, response["region"])
607
+ session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
608
+ region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
598
609
 
599
- bucket = Cnvrg::Helpers.decrypt(key,iv, response["bucket"])
610
+ bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
600
611
  is_s3 = response["is_s3"]
601
612
  if is_s3 or is_s3.nil?
602
613
  client = Aws::S3::Client.new(
603
- :access_key_id =>access,
614
+ :access_key_id => access,
604
615
  :secret_access_key => secret,
605
616
  :session_token => session,
606
617
  :region => region,
607
618
  :http_open_timeout => 60, :retry_limit => 20)
608
619
  else
609
- endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
620
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
610
621
  client = Aws::S3::Client.new(
611
- :access_key_id =>access,
622
+ :access_key_id => access,
612
623
  :secret_access_key => secret,
613
624
  :region => region,
614
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
625
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
615
626
  :http_open_timeout => 60, :retry_limit => 20)
616
627
  end
617
628
  list_files = response["files"]
@@ -628,18 +639,18 @@ module Cnvrg
628
639
  }
629
640
  download_count = 0
630
641
  Parallel.map((list_files), parallel_options) do |f|
631
- file_key = Cnvrg::Helpers.decrypt(key,iv, f["path"])
632
- begin
642
+ file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
633
643
  begin
634
- dir = File.dirname f["fullpath"]
635
- FileUtils.mkdir_p(dataset_home+"/"+ dir) unless File.exist? (dataset_home+"/"+ dir)
636
- end
644
+ begin
645
+ dir = File.dirname f["fullpath"]
646
+ FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
647
+ end
637
648
 
638
- File.open(dataset_home+"/"+f["fullpath"], 'w+') do |file|
639
- resp = client.get_object({bucket:bucket,
640
- key:file_key}, target: file)
641
- end
642
- download_count += 1
649
+ File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
650
+ resp = client.get_object({bucket: bucket,
651
+ key: file_key}, target: file)
652
+ end
653
+ download_count += 1
643
654
  rescue
644
655
  end
645
656
 
@@ -651,97 +662,95 @@ module Cnvrg
651
662
  end
652
663
 
653
664
 
654
-
655
-
656
665
  end
657
666
 
658
667
 
659
- def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
668
+ def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
660
669
  begin
661
- res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 ,new_version:true, as_link:as_link})
670
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
662
671
  Cnvrg::CLI.is_response_success(res, false)
663
- if res["result"]
664
- file_url = res["result"]["file_url"]
672
+ if res["result"]
673
+ file_url = res["result"]["file_url"]
665
674
 
666
- if as_link
667
- return res["result"]
668
- end
669
- # begin
670
- # if !Helpers.is_verify_ssl
671
- # tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
672
- #
673
- # else
674
- # tempfile = Down.download(file_url,open_timeout: 60)
675
- #
676
- # end
677
- #
678
- # FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
679
- # return true
680
- # rescue
681
- #
682
- # end
683
- download_resp = res
684
- filename = download_resp["result"]["filename"]
685
-
686
- absolute_path += ".conflict" if conflict
687
- sts_path = download_resp["result"]["path_sts"]
688
- retries = 0
689
- success= false
690
- while !success and retries < 20
691
- begin
692
- if !Helpers.is_verify_ssl
693
- body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
694
- else
695
- body = open(sts_path).read
696
- end
697
- success = true
698
- rescue => e
699
- retries +=1
700
- sleep(5)
675
+ if as_link
676
+ return res["result"]
677
+ end
678
+ # begin
679
+ # if !Helpers.is_verify_ssl
680
+ # tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
681
+ #
682
+ # else
683
+ # tempfile = Down.download(file_url,open_timeout: 60)
684
+ #
685
+ # end
686
+ #
687
+ # FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
688
+ # return true
689
+ # rescue
690
+ #
691
+ # end
692
+ download_resp = res
693
+ filename = download_resp["result"]["filename"]
701
694
 
695
+ absolute_path += ".conflict" if conflict
696
+ sts_path = download_resp["result"]["path_sts"]
697
+ retries = 0
698
+ success = false
699
+ while !success and retries < 20
700
+ begin
701
+ if !Helpers.is_verify_ssl
702
+ body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
703
+ else
704
+ body = open(sts_path).read
702
705
  end
706
+ success = true
707
+ rescue => e
708
+ retries += 1
709
+ sleep(5)
710
+
703
711
  end
704
- if !success
705
- return false
706
- end
707
- split = body.split("\n")
708
- key = split[0]
709
- iv = split[1]
712
+ end
713
+ if !success
714
+ return false
715
+ end
716
+ split = body.split("\n")
717
+ key = split[0]
718
+ iv = split[1]
710
719
 
711
- access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
720
+ access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
712
721
 
713
- secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
722
+ secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
714
723
 
715
- session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
716
- region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
724
+ session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
725
+ region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
717
726
 
718
- bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
719
- file_key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
727
+ bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
728
+ file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
720
729
 
721
- is_s3 = download_resp["result"]["is_s3"]
722
- if is_s3 or is_s3.nil?
723
- client = Aws::S3::Client.new(
724
- :access_key_id =>access,
725
- :secret_access_key => secret,
726
- :session_token => session,
727
- :region => region,
728
- :http_open_timeout => 60, :retry_limit => 20)
729
- else
730
- endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
731
- client = Aws::S3::Client.new(
732
- :access_key_id =>access,
733
- :secret_access_key => secret,
734
- :region => region,
735
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
736
- :http_open_timeout => 60, :retry_limit => 20)
737
- end
730
+ is_s3 = download_resp["result"]["is_s3"]
731
+ if is_s3 or is_s3.nil?
732
+ client = Aws::S3::Client.new(
733
+ :access_key_id => access,
734
+ :secret_access_key => secret,
735
+ :session_token => session,
736
+ :region => region,
737
+ :http_open_timeout => 60, :retry_limit => 20)
738
+ else
739
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
740
+ client = Aws::S3::Client.new(
741
+ :access_key_id => access,
742
+ :secret_access_key => secret,
743
+ :region => region,
744
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
745
+ :http_open_timeout => 60, :retry_limit => 20)
746
+ end
738
747
 
739
- File.open(project_home+"/"+absolute_path, 'w+') do |file|
740
- resp = client.get_object({bucket:bucket,
741
- key:file_key}, target: file)
742
- end
743
- return true
748
+ File.open(project_home + "/" + absolute_path, 'w+') do |file|
749
+ resp = client.get_object({bucket: bucket,
750
+ key: file_key}, target: file)
744
751
  end
752
+ return true
753
+ end
745
754
 
746
755
  rescue => e
747
756
  return false
@@ -751,7 +760,7 @@ module Cnvrg
751
760
 
752
761
  def download_data_file(commit_sha1, dataset_home)
753
762
  begin
754
- res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
763
+ res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
755
764
  Cnvrg::CLI.is_response_success(res, false)
756
765
  if res["result"]
757
766
  download_resp = res
@@ -768,38 +777,38 @@ module Cnvrg
768
777
  iv = split[1]
769
778
 
770
779
 
771
- access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
780
+ access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
772
781
 
773
- secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
782
+ secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
774
783
 
775
- session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
776
- region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
784
+ session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
785
+ region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
777
786
 
778
- bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
779
- file_key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
787
+ bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
788
+ file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
780
789
 
781
790
  is_s3 = download_resp["result"]["is_s3"]
782
791
  if is_s3 or is_s3.nil?
783
792
  client = Aws::S3::Client.new(
784
- :access_key_id =>access,
793
+ :access_key_id => access,
785
794
  :secret_access_key => secret,
786
795
  :session_token => session,
787
796
  :region => region,
788
797
  :http_open_timeout => 60, :retry_limit => 20)
789
798
  else
790
- endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
799
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
791
800
  client = Aws::S3::Client.new(
792
- :access_key_id =>access,
801
+ :access_key_id => access,
793
802
  :secret_access_key => secret,
794
803
  :region => region,
795
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
804
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
796
805
  :http_open_timeout => 60, :retry_limit => 20)
797
806
  end
798
807
 
799
808
 
800
- File.open(dataset_home+"/"+filename, 'w+') do |file|
809
+ File.open(dataset_home + "/" + filename, 'w+') do |file|
801
810
  resp = client.get_object({bucket: bucket,
802
- key: file_key }, target: file)
811
+ key: file_key}, target: file)
803
812
  end
804
813
  return filename
805
814
  end
@@ -810,7 +819,7 @@ module Cnvrg
810
819
  end
811
820
  end
812
821
 
813
- def download_file(absolute_path, relative_path, project_home, conflict=false)
822
+ def download_file(absolute_path, relative_path, project_home, conflict = false)
814
823
  res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
815
824
  Cnvrg::CLI.is_response_success(res, false)
816
825
  if res["result"]
@@ -848,7 +857,7 @@ module Cnvrg
848
857
  puts FileUtils.rmtree("#{absolute_path}")
849
858
  end
850
859
 
851
- def revoke_download_file(absolute_path, filename, conflict=false)
860
+ def revoke_download_file(absolute_path, filename, conflict = false)
852
861
  begin
853
862
  file_location = absolute_path.gsub(/#{filename}\/?$/, "")
854
863
 
@@ -859,6 +868,7 @@ module Cnvrg
859
868
  return false
860
869
  end
861
870
  end
871
+
862
872
  def revoke_download(tar_files, extracted_files)
863
873
  begin
864
874
 
@@ -872,32 +882,35 @@ module Cnvrg
872
882
  return true
873
883
 
874
884
  end
885
+
875
886
  def delete_commit(commit_sha1)
876
887
  response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
877
888
  Cnvrg::CLI.is_response_success(response, true)
878
889
  return response
879
890
  end
891
+
880
892
  def get_commit(commit_sha1)
881
893
  response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
882
894
  Cnvrg::CLI.is_response_success(response, true)
883
895
  return response
884
896
  end
885
897
 
886
- def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
898
+ def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
887
899
  begin
888
900
  #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
889
901
  # create a new commit.
890
902
  idx = (force || new_branch) ? {} : dataset.get_idx
891
903
  commit = idx[:commit]
892
904
  next_commit = idx[:next_commit]
893
- response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
894
- username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
905
+ response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
906
+ username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
895
907
  Cnvrg::CLI.is_response_success(response, true)
896
908
  return response
897
909
  rescue => e
898
910
  return false
899
911
  end
900
912
  end
913
+
901
914
  def last_valid_commit()
902
915
  begin
903
916
  #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -910,9 +923,9 @@ module Cnvrg
910
923
  end
911
924
  end
912
925
 
913
- def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
926
+ def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
914
927
  begin
915
- response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
928
+ response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
916
929
  Cnvrg::CLI.is_response_success(response, true)
917
930
  return response
918
931
  rescue => e
@@ -938,13 +951,13 @@ module Cnvrg
938
951
  end
939
952
 
940
953
  def get_trees(commit: "latest")
941
- response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
954
+ response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
942
955
  return nil unless Cnvrg::CLI.is_response_success(response, false)
943
956
  response['result']['files']
944
957
  end
945
958
 
946
959
  def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
947
- response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
960
+ response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
948
961
  unless Cnvrg::CLI.is_response_success(response, false)
949
962
  Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
950
963
  return nil
@@ -975,26 +988,26 @@ module Cnvrg
975
988
  end
976
989
  end
977
990
 
978
- def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
991
+ def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
979
992
  begin
980
- last_chunk_size = chunk_size
981
- q = { commit: commit, chunk_size: chunk_size}
982
- overall = 0
983
- while last_chunk_size > 0
984
- response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
985
- if Cnvrg::CLI.is_response_success(response, false)
986
- files = response['files']
987
- data_home = "#{Dir.pwd}/#{response['name']}"
988
- last_chunk_size = files['keys'].length
989
- break if last_chunk_size == 0
990
- res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
991
- overall += last_chunk_size
992
- q[:latest] = files['latest']
993
- else
994
- last_chunk_size = 0
993
+ last_chunk_size = chunk_size
994
+ q = {commit: commit, chunk_size: chunk_size}
995
+ overall = 0
996
+ while last_chunk_size > 0
997
+ response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
998
+ if Cnvrg::CLI.is_response_success(response, false)
999
+ files = response['files']
1000
+ data_home = "#{Dir.pwd}/#{response['name']}"
1001
+ last_chunk_size = files['keys'].length
1002
+ break if last_chunk_size == 0
1003
+ res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
1004
+ overall += last_chunk_size
1005
+ q[:latest] = files['latest']
1006
+ else
1007
+ last_chunk_size = 0
1008
+ end
995
1009
  end
996
- end
997
- Cnvrg::Result.new(true, "Cloned #{overall} files!")
1010
+ Cnvrg::Result.new(true, "Cloned #{overall} files!")
998
1011
  rescue => e
999
1012
  Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
1000
1013
 
@@ -1002,42 +1015,43 @@ module Cnvrg
1002
1015
  end
1003
1016
 
1004
1017
  def generate_parallel_idx
1005
-
1018
+
1006
1019
  end
1007
1020
 
1008
- def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
1021
+ def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
1009
1022
  begin
1023
+ refresh_storage_token
1010
1024
  parallel_options = {
1011
1025
  in_threads: ParallelThreads,
1012
1026
  isolation: true
1013
1027
  }
1014
1028
  Parallel.map(files["keys"], parallel_options) do |f|
1015
1029
  begin
1016
- local_path = @dataset.local_path + '/' + f['name']
1017
- Cnvrg::Logger.log_info("Downloading #{local_path}")
1018
- progressbar.progress += 1 if progressbar.present?
1019
- if local_path.end_with? "/"
1020
- @downloader.mkdir(local_path, recursive: true)
1021
- next
1022
- end
1030
+ local_path = @dataset.local_path + '/' + f['name']
1031
+ Cnvrg::Logger.log_info("Downloading #{local_path}")
1032
+ progressbar.progress += 1 if progressbar.present?
1033
+ if local_path.end_with? "/"
1034
+ @downloader.mkdir(local_path, recursive: true)
1035
+ next
1036
+ end
1023
1037
  # blob
1024
- local_path = "#{local_path}.conflict" if conflict
1025
- storage_path = f["path"]
1026
- if File.exists? local_path
1027
- Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1028
- next
1029
- end
1030
- resp = @downloader.download(storage_path, local_path)
1031
- Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
1038
+ local_path = "#{local_path}.conflict" if conflict
1039
+ storage_path = f["path"]
1040
+ if File.exists? local_path
1041
+ Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1042
+ next
1043
+ end
1044
+ resp = @downloader.download(storage_path, local_path)
1045
+ Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
1032
1046
  rescue => e
1033
1047
  Cnvrg::Logger.log_error(e)
1034
1048
  end
1035
1049
  end
1036
- return Cnvrg::Result.new(true,"Downloaded successfully")
1050
+ return Cnvrg::Result.new(true, "Downloaded successfully")
1037
1051
  rescue => e
1038
- Cnvrg::Logger.log_error(e)
1039
- return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
1040
- end
1052
+ Cnvrg::Logger.log_error(e)
1053
+ return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
1041
1054
  end
1055
+ end
1042
1056
  end
1043
1057
  end
@@ -108,7 +108,7 @@ module Cnvrg
108
108
  def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
109
109
  sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
110
110
  git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
111
- email_notification_error: false, email_notification_success: false)
111
+ email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
112
112
  response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
113
113
  commit_sha1: commit_to_run,
114
114
  instance_type: instance_type,
@@ -124,7 +124,8 @@ module Cnvrg
124
124
  restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
125
125
  prerun: prerun, requirements: requirements, recurring: recurring,
126
126
  email_notification_error: email_notification_error,
127
- email_notification_success: email_notification_success})
127
+ email_notification_success: email_notification_success,
128
+ emails_to_notify: emails_to_notify})
128
129
 
129
130
  return response
130
131
  end
@@ -7,7 +7,7 @@ require 'cnvrg/result'
7
7
  module Cnvrg
8
8
  class Files
9
9
  ParallelThreads = Cnvrg::Helpers.parallel_threads
10
- VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
10
+ VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
11
11
  LARGE_FILE=1024*1024*5
12
12
  MULTIPART_SPLIT=10000000
13
13
 
@@ -27,8 +27,16 @@ module Cnvrg
27
27
  @custom_progess = false
28
28
  @cli = cli
29
29
  @options = options
30
+ @token_issue_time = Time.current
30
31
  end
31
32
 
33
+ def refresh_storage_token
34
+ current_time = Time.current
35
+ if current_time - @token_issue_time > 3.hours
36
+ @client = @project.get_storage_client
37
+ @token_issue_time = Time.current
38
+ end
39
+ end
32
40
 
33
41
  def self.valid_file_name?(fullpath)
34
42
  VALID_FILE_NAME.match(fullpath).blank?
@@ -97,7 +105,13 @@ module Cnvrg
97
105
  files = res['files']
98
106
 
99
107
  #upload files
108
+ token_mutex = Mutex.new
100
109
  blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
110
+
111
+ token_mutex.synchronize {
112
+ refresh_storage_token
113
+ }
114
+
101
115
  begin
102
116
  Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
103
117
  rescue => e
@@ -721,8 +735,15 @@ module Cnvrg
721
735
  in_threads: Cnvrg::Helpers.parallel_threads,
722
736
  isolation: true
723
737
  }
738
+
739
+ token_mutex = Mutex.new
740
+
724
741
  Parallel.map(files["keys"], parallel_options) do |f|
725
742
 
743
+ token_mutex.synchronize {
744
+ refresh_storage_token
745
+ }
746
+
726
747
  file_path = f["name"]
727
748
  if file_path.end_with? "/"
728
749
  # dir
@@ -370,6 +370,13 @@ module Cnvrg
370
370
  File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
371
371
  end
372
372
 
373
+ def generate_git_diff
374
+ git_diff = `git diff --name-only`
375
+ git_diff.split("\n")
376
+ rescue
377
+ []
378
+ end
379
+
373
380
  def generate_output_dir(output_dir)
374
381
  Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
375
382
  upload_list = []
@@ -387,7 +394,7 @@ module Cnvrg
387
394
  upload_list << output_dir + "/"
388
395
  end
389
396
  Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
390
- return upload_list
397
+ upload_list
391
398
  end
392
399
  def generate_output_dir_tmp(output_dir)
393
400
  upload_list = []
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.6.31'
2
+ VERSION = '1.6.38'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.31
4
+ version: 1.6.38
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-04-01 00:00:00.000000000 Z
13
+ date: 2020-06-23 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler