cnvrg 1.6.31 → 1.6.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: d7dee30d3b567e7e1ab181aed63426524d6f04459fe365ce5d60cc4c7671ed03
4
- data.tar.gz: f8fcb9901d5bc06471500246a63f3ff28d5593fe6564f8cf0930d6d41342ed16
3
+ metadata.gz: e5953510af9633c925be99ac1463aef31667eed5f83f5b0c610672ab9fe375cb
4
+ data.tar.gz: 7bd8fce2d93a20ed695e4c202f2b29092de84bf90332385a3921844046a0c591
5
5
  SHA512:
6
- metadata.gz: 935d27a9023559e57694c726515846175bcd892a1906fdf058983268ba2e2b81ecc41bf8b31a689b01302cdc8afaef8dd6d5130d686857d6cfe30e4fdec3f45a
7
- data.tar.gz: 9ebd4da0e91c62e64712453eab68c5fe7914dfc6ec5ad26aae9f6fbbf6f1cdfce8d187f7d8f8a1c0aa5f1955c6842ae317da9fca2238d20c8410c3d6c31158a3
6
+ metadata.gz: 8b93cd899060800095895967dbe2b8f3391e2d77b6f29ea05cd6651de4d26de2ba24fdaedaa8dafad82a22e250d95abf7651d2b77afe2dcf68b883461774c965
7
+ data.tar.gz: a4dc324e6f58b628013bb00278a74233f6b2c6e4ffa260e9edf74ea2557ca144907449078defb1abdb22ffe411fdde196b2e029bd66e67233140cf54d6caf67d
@@ -1184,7 +1184,7 @@ module Cnvrg
1184
1184
  end
1185
1185
 
1186
1186
  desc '', '', :hide => true
1187
- def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000)
1187
+ def data_put(dataset_url, files: [], dir: '', commit: '', chunk_size: 1000, message: nil)
1188
1188
  begin
1189
1189
  verify_logged_in(false)
1190
1190
  log_start(__method__, args, options)
@@ -1204,7 +1204,7 @@ module Cnvrg
1204
1204
  log_message("Uploading #{@files.size} files", Thor::Shell::Color::GREEN)
1205
1205
  number_of_chunks = (@files.size.to_f / chunk_size).ceil
1206
1206
  if commit.blank?
1207
- response = @datafiles.start_commit(false, true, chunks: number_of_chunks)
1207
+ response = @datafiles.start_commit(false, true, chunks: number_of_chunks, message: message )
1208
1208
  unless response #means we failed in the start commit.
1209
1209
  raise SignalException.new(1, "Cant put files into dataset, check the dataset id")
1210
1210
  end
@@ -1837,8 +1837,6 @@ module Cnvrg
1837
1837
  end
1838
1838
  clone_resp = Project.clone_dir(slug, owner, project_name,git)
1839
1839
  project_home = Dir.pwd + "/" + project_name
1840
-
1841
-
1842
1840
  end
1843
1841
 
1844
1842
  if clone_resp
@@ -2213,6 +2211,7 @@ module Cnvrg
2213
2211
  method_option :return_id, :type => :boolean, :aliases => ["-r", "--return_id"], :default => false
2214
2212
  method_option :files, :type => :string, :aliases => ["--files"], :default => nil
2215
2213
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2214
+ method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
2216
2215
  method_option :job_slug, :type => :string, :aliases => ["--job"], :default => nil, :hide=>true
2217
2216
  method_option :job_type, :type => :string, :aliases => [ "--job_type"], :default => nil, :hide=>true
2218
2217
 
@@ -2237,19 +2236,21 @@ module Cnvrg
2237
2236
  spec_files_to_upload = spec_files_to_upload.split(",")
2238
2237
  end
2239
2238
  if @project.is_git
2239
+ list = []
2240
2240
  git_output_dir = options["output_dir"] || output_dir
2241
2241
  if git_output_dir.present?
2242
2242
  if git_output_dir.ends_with? "/"
2243
2243
  git_output_dir = git_output_dir[0..-2]
2244
2244
  end
2245
2245
  list = @project.generate_output_dir(git_output_dir)
2246
- spec_files_to_upload = list
2247
- if spec_files_to_upload.blank?
2248
- log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
2249
- return true
2250
- end
2251
- force = true
2252
2246
  end
2247
+ list += @project.generate_git_diff if options["git_diff"]
2248
+ spec_files_to_upload = list
2249
+ if spec_files_to_upload.blank?
2250
+ log_message("#{check} Project is up to date", Thor::Shell::Color::GREEN, (((options["sync"] or sync) and !direct) ? false : true))
2251
+ return true
2252
+ end
2253
+ force = true
2253
2254
  end
2254
2255
 
2255
2256
  if ignore.nil? or ignore.empty?
@@ -2894,6 +2895,7 @@ module Cnvrg
2894
2895
  method_option :job_type, :type => :string, :aliases => ["-jt", "--job_type"], :default => nil
2895
2896
  method_option :files, :type => :string, :aliases => ["--files"], :default => nil
2896
2897
  method_option :output_dir, :type => :string, :aliases => ["--output_dir"], :default => nil
2898
+ method_option :git_diff, :type => :boolean, :aliases => ["--git_diff"], :default => false
2897
2899
  def sync(direct = true)
2898
2900
  verify_logged_in(true) if direct
2899
2901
  @project = Project.new(get_project_home)
@@ -2914,7 +2916,7 @@ module Cnvrg
2914
2916
  end
2915
2917
  invoke :upload, [false, true, direct, "",in_exp,options[:force], options["output_dir"],job_type, job_slug ], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
2916
2918
  :ignore => options[:ignore], :force => options[:force], :message => options[:message], :deploy => options["deploy"], :return_id => options["return_id"],
2917
- :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type
2919
+ :files => options["files"], :output_dir => options["output_dir"], :job_slug => job_slug, :job_type => job_type, :git_diff=> options["git_diff"]
2918
2920
  end
2919
2921
 
2920
2922
  desc 'run cmd', 'Runs an experiment'
@@ -2956,6 +2958,7 @@ module Cnvrg
2956
2958
  method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
2957
2959
  method_option :notify_on_error, :type => :boolean, :aliases => ["-noe", "--notify_on_error"], :default => nil
2958
2960
  method_option :notify_on_success, :type => :boolean, :aliases => ["-nos", "--notify_on_success"], :default => nil
2961
+ method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error, comma separated"
2959
2962
 
2960
2963
  def run(*cmd)
2961
2964
  verify_logged_in(true)
@@ -2989,6 +2992,7 @@ module Cnvrg
2989
2992
  requirements = options["requirements"]
2990
2993
  email_notification_error = options["notify_on_error"]
2991
2994
  email_notification_success = options["notify_on_success"]
2995
+ emails = options["emails"]
2992
2996
 
2993
2997
  if !data.present? and data_query.present?
2994
2998
  log_message("Please provide data with data_query", Thor::Shell::Color::RED)
@@ -3003,7 +3007,6 @@ module Cnvrg
3003
3007
  restart_if_stuck = options["restart_if_stuck"]
3004
3008
 
3005
3009
  options_hash = Hash[options]
3006
-
3007
3010
  if local
3008
3011
  if Cnvrg::Helpers.windows?
3009
3012
  say "Windows is currently not supported for running experiments locally"
@@ -3039,7 +3042,7 @@ module Cnvrg
3039
3042
  :periodic_sync => periodic_sync, :dataset_only_tree=> dataset_only_tree,
3040
3043
  :output_dir=>output_dir, :data_query=>data_query, :git_commit =>git_commit, :git_branch=> git_branch,
3041
3044
  :restart_if_stuck =>restart_if_stuck, :local_folders => local_folders, :datasets => datasets, :prerun => prerun, :requirements => requirements,
3042
- :email_notification_error => email_notification_error, :email_notification_success => email_notification_success
3045
+ :email_notification_error => email_notification_error, :email_notification_success => email_notification_success, :emails => emails
3043
3046
  return
3044
3047
  end
3045
3048
 
@@ -3345,8 +3348,9 @@ module Cnvrg
3345
3348
  method_option :datasets, :type => :string, :aliases => ["--datasets"], :default => nil
3346
3349
  method_option :prerun, :type => :boolean, :aliases => ["-p", "--prerun"], :default => true
3347
3350
  method_option :requirements, :type => :boolean, :aliases => ["-r", "--requirements"], :default => true
3348
- method_option :email_notification_error, :type => :boolean, :aliases => ["--email_notification_error"], :default => true
3349
- method_option :email_notification_success, :type => :boolean, :aliases => ["--email_notification_success"], :default => true
3351
+ method_option :email_notification_error, :type => :boolean, :aliases => ["-noe", "--email_notification_error"], :default => true
3352
+ method_option :email_notification_success, :type => :boolean, :aliases => ["-nos", "--email_notification_success"], :default => true
3353
+ method_option :emails, :type => :string, :aliases => ["-es", "--emails"], :default => "", :desc => "additional emails to notify on success / or error"
3350
3354
 
3351
3355
  def exec_remote(*cmd)
3352
3356
 
@@ -3368,6 +3372,7 @@ module Cnvrg
3368
3372
  requirements = options["requirements"]
3369
3373
  email_notification_error = options["email_notification_error"]
3370
3374
  email_notification_success = options["email_notification_success"]
3375
+ emails = options["emails"]
3371
3376
  max_time = options["max_time"]
3372
3377
  if !max_time.nil? and !max_time.empty?
3373
3378
  max_time = max_time.to_i
@@ -3407,7 +3412,7 @@ module Cnvrg
3407
3412
  options_hash.except!("schedule", "recurring", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "title",
3408
3413
  "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl","max_time","dataset_only_tree",
3409
3414
  "data_query", "git_commit","git_branch", "restart_if_stuck","local_folders","output_dir", "commit", "datasets", "requirements", "prerun",
3410
- "email_notification_error", "email_notification_success")
3415
+ "email_notification_error", "email_notification_success", "emails")
3411
3416
  exec_options = options_hash.map {|x| "--#{x[0]}=#{x[1]}"}.flatten.join(" ")
3412
3417
  command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
3413
3418
  commit_to_run = options["commit"] || nil
@@ -3475,7 +3480,7 @@ module Cnvrg
3475
3480
  res = exp.exec_remote(command, commit_to_run, instance_type, image, schedule, local_timestamp, grid, path_to_cmd, data, data_commit,
3476
3481
  periodic_sync, sync_before_terminate, max_time, ds_sync_options,output_dir,
3477
3482
  data_query, git_commit, git_branch, restart_if_stuck,local_folders_options, title, datasets, prerun: prerun, requirements: requirements, recurring: recurring,
3478
- email_notification_error: email_notification_error, email_notification_success: email_notification_success)
3483
+ email_notification_error: email_notification_error, email_notification_success: email_notification_success, emails_to_notify: emails)
3479
3484
  if Cnvrg::CLI.is_response_success(res)
3480
3485
  check = Helpers.checkmark()
3481
3486
  str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
@@ -186,11 +186,14 @@ module Cnvrg
186
186
  desc 'data put DATASET_URL FILES_PREFIX', 'Upload selected files from local dataset directory to remote server'
187
187
  method_option :dir, :type => :string, :aliases => ["-d", "--dir"], :default => ''
188
188
  method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ''
189
+ method_option :message, :type => :string, :aliases => ["--message"], :desc => "create commit with message", :default => nil
189
190
  def put(dataset_url, *files)
190
191
  cli = Cnvrg::CLI.new()
191
192
  dir = options[:dir]
192
193
  commit = options[:commit]
193
- cli.data_put(dataset_url, files: files, dir: dir, commit: commit)
194
+ message = options[:message]
195
+
196
+ cli.data_put(dataset_url, files: files, dir: dir, commit: commit, message: message)
194
197
  end
195
198
 
196
199
  desc 'data clone_query --query=QUERY_SLUG DATASET_URL', 'Clone dataset with specific query'
@@ -9,8 +9,8 @@ module Cnvrg
9
9
  class Datafiles
10
10
  ParallelThreads ||= Cnvrg::Helpers.parallel_threads
11
11
 
12
- LARGE_FILE=1024*1024*5
13
- MULTIPART_SPLIT=10000000
12
+ LARGE_FILE = 1024 * 1024 * 5
13
+ MULTIPART_SPLIT = 10000000
14
14
 
15
15
  attr_reader :base_resource
16
16
 
@@ -20,11 +20,20 @@ module Cnvrg
20
20
  @dataset = dataset
21
21
  @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
22
22
  @downloader = @dataset.get_storage_client
23
+ @token_issue_time = Time.current
24
+ end
25
+
26
+ def refresh_storage_token
27
+ current_time = Time.current
28
+ if current_time - @token_issue_time > 3.hours
29
+ @downloader = @dataset.get_storage_client
30
+ @token_issue_time = Time.current
31
+ end
23
32
  end
24
33
 
25
34
  def check_file_sha1(filename, org_sha1, tag: 'conflict')
26
35
  file_loc = "#{Dir.pwd}/#{filename}"
27
- sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
36
+ sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
28
37
  return 0 if sha1 == org_sha1
29
38
  FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
30
39
  return 1
@@ -43,7 +52,7 @@ module Cnvrg
43
52
  end
44
53
  raise SignalException.new(1, "Cant find file #{file}") unless File.exists? "#{Dir.pwd}/#{file}"
45
54
  end
46
- paths.map{|p| p.gsub(/^\.\//, '')}
55
+ paths.map { |p| p.gsub(/^\.\//, '') }
47
56
  end
48
57
 
49
58
  def check_files_sha1(files, resolver, tag)
@@ -74,11 +83,11 @@ module Cnvrg
74
83
  prefix = @dataset.get_dataset["bucket_prefix"]
75
84
  batch_size = 10000
76
85
  pbar = ProgressBar.create(:title => "Download Progress",
77
- :progress_mark => '=',
78
- :format => "%b%i| %c Files downloaded",
79
- :starting_at => 0,
80
- :total => nil,
81
- :autofinish => true)
86
+ :progress_mark => '=',
87
+ :format => "%b%i| %c Files downloaded",
88
+ :starting_at => 0,
89
+ :total => nil,
90
+ :autofinish => true)
82
91
  parallel_options = {
83
92
  in_threads: ParallelThreads,
84
93
  in_processes: Cnvrg::CLI::ParallelProcesses,
@@ -107,7 +116,7 @@ module Cnvrg
107
116
  if response.present?
108
117
  msg = response['result']
109
118
  else
110
- msg = "Cant save changes in the dataset"
119
+ msg = "Cant save changes in the dataset"
111
120
  end
112
121
 
113
122
  Cnvrg::Result.new(Cnvrg::CLI.is_response_success(response, false), msg)
@@ -117,6 +126,7 @@ module Cnvrg
117
126
  def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, prefix: '', partial_commit: nil, total: nil)
118
127
  begin
119
128
  Cnvrg::Logger.log_info("Sending Upload Files request")
129
+ refresh_storage_token
120
130
  error = nil
121
131
  upload_resp = nil
122
132
  10.times do
@@ -161,9 +171,9 @@ module Cnvrg
161
171
  end
162
172
  @temp_upload_progressbar.progress += 1 if @temp_upload_progressbar.present?
163
173
  end
164
- blob_ids = files.values.map {|f| f['bv_id']}
174
+ blob_ids = files.values.map { |f| f['bv_id'] }
165
175
  if blob_ids.present?
166
- dirs = tree.keys.select {|k| tree[k].nil?} || []
176
+ dirs = tree.keys.select { |k| tree[k].nil? } || []
167
177
  Cnvrg::Logger.info("Sending Upload files save")
168
178
  upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
169
179
  unless Cnvrg::CLI.is_response_success(upload_resp, false)
@@ -198,12 +208,12 @@ module Cnvrg
198
208
  file_size = File.size(absolute_path).to_f
199
209
  mime_type = MimeMagic.by_path(absolute_path)
200
210
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
201
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
211
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
202
212
  if (absolute_path.include? "_tags.yml" or absolute_path.include? "_tags.yaml")
203
213
  is_valid = false
204
214
  begin
205
- content = open(absolute_path).read()
206
- hash = YAML.load(open(absolute_path).read())
215
+ content = open(absolute_path).read()
216
+ hash = YAML.load(open(absolute_path).read())
207
217
  # if level 1 keys count is 1
208
218
  if hash.keys.count == 1
209
219
  if hash["tags"].present?
@@ -211,7 +221,7 @@ module Cnvrg
211
221
  elsif hash[hash.keys.first].class != Hash
212
222
  is_valid = true
213
223
  end
214
- # if level 1 keys count is greater than 1
224
+ # if level 1 keys count is greater than 1
215
225
  elsif hash.keys.count > 1
216
226
  if hash["tags"].present? and hash["tags"].class == Hash
217
227
  is_valid = false
@@ -245,6 +255,7 @@ module Cnvrg
245
255
  return s3_res
246
256
  end
247
257
  end
258
+
248
259
  def upload_tar_file(absolute_path, relative_path, commit_sha1)
249
260
  begin
250
261
  file_name = File.basename relative_path
@@ -252,24 +263,24 @@ module Cnvrg
252
263
  mime_type = MimeMagic.by_path(absolute_path)
253
264
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
254
265
  begin
255
- chunked_bytes = [100, (file_size*0.01)].min
266
+ chunked_bytes = [100, (file_size * 0.01)].min
256
267
  total_yanked = ""
257
268
  open(absolute_path, "rb") do |f|
258
269
  total_yanked = f.read(chunked_bytes)
259
270
  end
260
271
  if !total_yanked.empty?
261
- sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
272
+ sha1 = OpenSSL::Digest::SHA1.hexdigest(total_yanked)
262
273
  else
263
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
274
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
264
275
  end
265
276
  rescue
266
- sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
277
+ sha1 = OpenSSL::Digest::SHA1.file(absolute_path).hexdigest
267
278
  end
268
279
 
269
280
  upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
270
281
  commit_sha1: commit_sha1, file_name: file_name,
271
282
  file_size: file_size, file_content_type: content_type, sha1: sha1,
272
- new_version:true})
283
+ new_version: true})
273
284
  if Cnvrg::CLI.is_response_success(upload_resp, false)
274
285
  path = upload_resp["result"]["path"]
275
286
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -305,14 +316,15 @@ module Cnvrg
305
316
  return false
306
317
 
307
318
  end
308
- def upload_data_log_file(absolute_path, relative_path,data_commit_sha)
319
+
320
+ def upload_data_log_file(absolute_path, relative_path, data_commit_sha)
309
321
  file_name = File.basename relative_path
310
322
  file_size = File.size(absolute_path).to_f
311
323
  content_type = "text/x-log"
312
324
  upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_data_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
313
- file_name: file_name, log_date: Time.now,
314
- file_size: file_size, file_content_type: content_type,
315
- data_commit_sha1:data_commit_sha})
325
+ file_name: file_name, log_date: Time.now,
326
+ file_size: file_size, file_content_type: content_type,
327
+ data_commit_sha1: data_commit_sha})
316
328
  if Cnvrg::CLI.is_response_success(upload_resp, false)
317
329
  path = upload_resp["result"]["path"]
318
330
  s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
@@ -403,15 +415,15 @@ module Cnvrg
403
415
 
404
416
  end
405
417
 
406
- def upload_cnvrg_image(absolute_path, image_name,secret)
418
+ def upload_cnvrg_image(absolute_path, image_name, secret)
407
419
  file_name = File.basename relative_path
408
420
  file_size = File.size(absolute_path).to_f
409
421
  mime_type = MimeMagic.by_path(absolute_path)
410
422
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
411
423
  upload_resp = Cnvrg::API.request("images/#{image_name}/upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: absolute_path,
412
- file_name: file_name,
413
- file_size: file_size, file_content_type: content_type,
414
- secret: secret})
424
+ file_name: file_name,
425
+ file_size: file_size, file_content_type: content_type,
426
+ secret: secret})
415
427
  if Cnvrg::CLI.is_response_success(upload_resp, false)
416
428
  path = upload_resp["result"]["path"]
417
429
  s3_res = upload_large_files_s3(upload_resp, absolute_path)
@@ -455,7 +467,7 @@ module Cnvrg
455
467
  sts_path = upload_resp["result"]["path_sts"]
456
468
 
457
469
  retries = 0
458
- success= false
470
+ success = false
459
471
  while !success and retries < 20
460
472
  begin
461
473
  if !Helpers.is_verify_ssl
@@ -465,7 +477,7 @@ module Cnvrg
465
477
  end
466
478
  success = true
467
479
  rescue => e
468
- retries +=1
480
+ retries += 1
469
481
  sleep(5)
470
482
 
471
483
  end
@@ -477,33 +489,33 @@ module Cnvrg
477
489
  key = split[0]
478
490
  iv = split[1]
479
491
  is_s3 = upload_resp["result"]["is_s3"]
480
- access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
492
+ access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
481
493
 
482
- secret = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
494
+ secret = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_s"])
483
495
 
484
- session = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
485
- region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
496
+ session = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_st"])
497
+ region = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["region"])
486
498
 
487
- bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
488
- server_side_encryption =upload_resp["result"]["server_side_encryption"]
499
+ bucket = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["bucket"])
500
+ server_side_encryption = upload_resp["result"]["server_side_encryption"]
489
501
  use_accelerate_endpoint = false
490
502
  if is_s3 or is_s3.nil?
491
503
 
492
- use_accelerate_endpoint =true
504
+ use_accelerate_endpoint = true
493
505
  client = Aws::S3::Client.new(
494
- :access_key_id =>access,
506
+ :access_key_id => access,
495
507
  :secret_access_key => secret,
496
508
  :session_token => session,
497
509
  :region => region,
498
510
  :http_open_timeout => 60, :retry_limit => 20)
499
511
  else
500
- endpoint = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["endpoint"])
512
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["endpoint"])
501
513
  use_accelerate_endpoint = false
502
514
  client = Aws::S3::Client.new(
503
- :access_key_id =>access,
515
+ :access_key_id => access,
504
516
  :secret_access_key => secret,
505
517
  :region => region,
506
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
518
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
507
519
  :http_open_timeout => 60, :retry_limit => 20)
508
520
  end
509
521
 
@@ -515,13 +527,11 @@ module Cnvrg
515
527
  end
516
528
 
517
529
 
518
-
519
-
520
530
  s3 = Aws::S3::Resource.new(client: client)
521
531
 
522
532
  resp = s3.bucket(bucket).
523
- object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
524
- upload_file(file_path,options)
533
+ object(upload_resp["result"]["path"] + "/" + File.basename(file_path)).
534
+ upload_file(file_path, options)
525
535
 
526
536
 
527
537
  return resp
@@ -578,6 +588,7 @@ module Cnvrg
578
588
  response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
579
589
  return Cnvrg::CLI.is_response_success(response, false)
580
590
  end
591
+
581
592
  def download_list_files_in_query(response, dataset_home)
582
593
  sts_path = response["path_sts"]
583
594
  if !Helpers.is_verify_ssl
@@ -589,29 +600,29 @@ module Cnvrg
589
600
  key = split[0]
590
601
  iv = split[1]
591
602
 
592
- access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
603
+ access = Cnvrg::Helpers.decrypt(key, iv, response["sts_a"])
593
604
 
594
- secret = Cnvrg::Helpers.decrypt(key,iv, response["sts_s"])
605
+ secret = Cnvrg::Helpers.decrypt(key, iv, response["sts_s"])
595
606
 
596
- session = Cnvrg::Helpers.decrypt(key,iv, response["sts_st"])
597
- region = Cnvrg::Helpers.decrypt(key,iv, response["region"])
607
+ session = Cnvrg::Helpers.decrypt(key, iv, response["sts_st"])
608
+ region = Cnvrg::Helpers.decrypt(key, iv, response["region"])
598
609
 
599
- bucket = Cnvrg::Helpers.decrypt(key,iv, response["bucket"])
610
+ bucket = Cnvrg::Helpers.decrypt(key, iv, response["bucket"])
600
611
  is_s3 = response["is_s3"]
601
612
  if is_s3 or is_s3.nil?
602
613
  client = Aws::S3::Client.new(
603
- :access_key_id =>access,
614
+ :access_key_id => access,
604
615
  :secret_access_key => secret,
605
616
  :session_token => session,
606
617
  :region => region,
607
618
  :http_open_timeout => 60, :retry_limit => 20)
608
619
  else
609
- endpoint = Cnvrg::Helpers.decrypt(key,iv, response["endpoint_url"])
620
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, response["endpoint_url"])
610
621
  client = Aws::S3::Client.new(
611
- :access_key_id =>access,
622
+ :access_key_id => access,
612
623
  :secret_access_key => secret,
613
624
  :region => region,
614
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
625
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
615
626
  :http_open_timeout => 60, :retry_limit => 20)
616
627
  end
617
628
  list_files = response["files"]
@@ -628,18 +639,18 @@ module Cnvrg
628
639
  }
629
640
  download_count = 0
630
641
  Parallel.map((list_files), parallel_options) do |f|
631
- file_key = Cnvrg::Helpers.decrypt(key,iv, f["path"])
632
- begin
642
+ file_key = Cnvrg::Helpers.decrypt(key, iv, f["path"])
633
643
  begin
634
- dir = File.dirname f["fullpath"]
635
- FileUtils.mkdir_p(dataset_home+"/"+ dir) unless File.exist? (dataset_home+"/"+ dir)
636
- end
644
+ begin
645
+ dir = File.dirname f["fullpath"]
646
+ FileUtils.mkdir_p(dataset_home + "/" + dir) unless File.exist? (dataset_home + "/" + dir)
647
+ end
637
648
 
638
- File.open(dataset_home+"/"+f["fullpath"], 'w+') do |file|
639
- resp = client.get_object({bucket:bucket,
640
- key:file_key}, target: file)
641
- end
642
- download_count += 1
649
+ File.open(dataset_home + "/" + f["fullpath"], 'w+') do |file|
650
+ resp = client.get_object({bucket: bucket,
651
+ key: file_key}, target: file)
652
+ end
653
+ download_count += 1
643
654
  rescue
644
655
  end
645
656
 
@@ -651,97 +662,95 @@ module Cnvrg
651
662
  end
652
663
 
653
664
 
654
-
655
-
656
665
  end
657
666
 
658
667
 
659
- def download_file_s3(absolute_path, relative_path, project_home, conflict=false, commit_sha1=nil, as_link=false)
668
+ def download_file_s3(absolute_path, relative_path, project_home, conflict = false, commit_sha1 = nil, as_link = false)
660
669
  begin
661
- res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1 ,new_version:true, as_link:as_link})
670
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1, new_version: true, as_link: as_link})
662
671
  Cnvrg::CLI.is_response_success(res, false)
663
- if res["result"]
664
- file_url = res["result"]["file_url"]
672
+ if res["result"]
673
+ file_url = res["result"]["file_url"]
665
674
 
666
- if as_link
667
- return res["result"]
668
- end
669
- # begin
670
- # if !Helpers.is_verify_ssl
671
- # tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
672
- #
673
- # else
674
- # tempfile = Down.download(file_url,open_timeout: 60)
675
- #
676
- # end
677
- #
678
- # FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
679
- # return true
680
- # rescue
681
- #
682
- # end
683
- download_resp = res
684
- filename = download_resp["result"]["filename"]
685
-
686
- absolute_path += ".conflict" if conflict
687
- sts_path = download_resp["result"]["path_sts"]
688
- retries = 0
689
- success= false
690
- while !success and retries < 20
691
- begin
692
- if !Helpers.is_verify_ssl
693
- body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
694
- else
695
- body = open(sts_path).read
696
- end
697
- success = true
698
- rescue => e
699
- retries +=1
700
- sleep(5)
675
+ if as_link
676
+ return res["result"]
677
+ end
678
+ # begin
679
+ # if !Helpers.is_verify_ssl
680
+ # tempfile = Down.download(file_url,open_timeout: 60,ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE )
681
+ #
682
+ # else
683
+ # tempfile = Down.download(file_url,open_timeout: 60)
684
+ #
685
+ # end
686
+ #
687
+ # FileUtils.move(tempfile.path, project_home+"/"+ absolute_path)
688
+ # return true
689
+ # rescue
690
+ #
691
+ # end
692
+ download_resp = res
693
+ filename = download_resp["result"]["filename"]
701
694
 
695
+ absolute_path += ".conflict" if conflict
696
+ sts_path = download_resp["result"]["path_sts"]
697
+ retries = 0
698
+ success = false
699
+ while !success and retries < 20
700
+ begin
701
+ if !Helpers.is_verify_ssl
702
+ body = open(sts_path, {ssl_verify_mode: OpenSSL::SSL::VERIFY_NONE}).read
703
+ else
704
+ body = open(sts_path).read
702
705
  end
706
+ success = true
707
+ rescue => e
708
+ retries += 1
709
+ sleep(5)
710
+
703
711
  end
704
- if !success
705
- return false
706
- end
707
- split = body.split("\n")
708
- key = split[0]
709
- iv = split[1]
712
+ end
713
+ if !success
714
+ return false
715
+ end
716
+ split = body.split("\n")
717
+ key = split[0]
718
+ iv = split[1]
710
719
 
711
- access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
720
+ access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
712
721
 
713
- secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
722
+ secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
714
723
 
715
- session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
716
- region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
724
+ session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
725
+ region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
717
726
 
718
- bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
719
- file_key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
727
+ bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
728
+ file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
720
729
 
721
- is_s3 = download_resp["result"]["is_s3"]
722
- if is_s3 or is_s3.nil?
723
- client = Aws::S3::Client.new(
724
- :access_key_id =>access,
725
- :secret_access_key => secret,
726
- :session_token => session,
727
- :region => region,
728
- :http_open_timeout => 60, :retry_limit => 20)
729
- else
730
- endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
731
- client = Aws::S3::Client.new(
732
- :access_key_id =>access,
733
- :secret_access_key => secret,
734
- :region => region,
735
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
736
- :http_open_timeout => 60, :retry_limit => 20)
737
- end
730
+ is_s3 = download_resp["result"]["is_s3"]
731
+ if is_s3 or is_s3.nil?
732
+ client = Aws::S3::Client.new(
733
+ :access_key_id => access,
734
+ :secret_access_key => secret,
735
+ :session_token => session,
736
+ :region => region,
737
+ :http_open_timeout => 60, :retry_limit => 20)
738
+ else
739
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
740
+ client = Aws::S3::Client.new(
741
+ :access_key_id => access,
742
+ :secret_access_key => secret,
743
+ :region => region,
744
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
745
+ :http_open_timeout => 60, :retry_limit => 20)
746
+ end
738
747
 
739
- File.open(project_home+"/"+absolute_path, 'w+') do |file|
740
- resp = client.get_object({bucket:bucket,
741
- key:file_key}, target: file)
742
- end
743
- return true
748
+ File.open(project_home + "/" + absolute_path, 'w+') do |file|
749
+ resp = client.get_object({bucket: bucket,
750
+ key: file_key}, target: file)
744
751
  end
752
+ return true
753
+ end
745
754
 
746
755
  rescue => e
747
756
  return false
@@ -751,7 +760,7 @@ module Cnvrg
751
760
 
752
761
  def download_data_file(commit_sha1, dataset_home)
753
762
  begin
754
- res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
763
+ res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1, new_version: true})
755
764
  Cnvrg::CLI.is_response_success(res, false)
756
765
  if res["result"]
757
766
  download_resp = res
@@ -768,38 +777,38 @@ module Cnvrg
768
777
  iv = split[1]
769
778
 
770
779
 
771
- access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
780
+ access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
772
781
 
773
- secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
782
+ secret = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_s"])
774
783
 
775
- session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
776
- region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
784
+ session = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_st"])
785
+ region = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["region"])
777
786
 
778
- bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
779
- file_key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
787
+ bucket = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["bucket"])
788
+ file_key = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["key"])
780
789
 
781
790
  is_s3 = download_resp["result"]["is_s3"]
782
791
  if is_s3 or is_s3.nil?
783
792
  client = Aws::S3::Client.new(
784
- :access_key_id =>access,
793
+ :access_key_id => access,
785
794
  :secret_access_key => secret,
786
795
  :session_token => session,
787
796
  :region => region,
788
797
  :http_open_timeout => 60, :retry_limit => 20)
789
798
  else
790
- endpoint = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["endpoint_url"])
799
+ endpoint = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["endpoint_url"])
791
800
  client = Aws::S3::Client.new(
792
- :access_key_id =>access,
801
+ :access_key_id => access,
793
802
  :secret_access_key => secret,
794
803
  :region => region,
795
- :endpoint=> endpoint,:force_path_style=> true,:ssl_verify_peer=>false,
804
+ :endpoint => endpoint, :force_path_style => true, :ssl_verify_peer => false,
796
805
  :http_open_timeout => 60, :retry_limit => 20)
797
806
  end
798
807
 
799
808
 
800
- File.open(dataset_home+"/"+filename, 'w+') do |file|
809
+ File.open(dataset_home + "/" + filename, 'w+') do |file|
801
810
  resp = client.get_object({bucket: bucket,
802
- key: file_key }, target: file)
811
+ key: file_key}, target: file)
803
812
  end
804
813
  return filename
805
814
  end
@@ -810,7 +819,7 @@ module Cnvrg
810
819
  end
811
820
  end
812
821
 
813
- def download_file(absolute_path, relative_path, project_home, conflict=false)
822
+ def download_file(absolute_path, relative_path, project_home, conflict = false)
814
823
  res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
815
824
  Cnvrg::CLI.is_response_success(res, false)
816
825
  if res["result"]
@@ -848,7 +857,7 @@ module Cnvrg
848
857
  puts FileUtils.rmtree("#{absolute_path}")
849
858
  end
850
859
 
851
- def revoke_download_file(absolute_path, filename, conflict=false)
860
+ def revoke_download_file(absolute_path, filename, conflict = false)
852
861
  begin
853
862
  file_location = absolute_path.gsub(/#{filename}\/?$/, "")
854
863
 
@@ -859,6 +868,7 @@ module Cnvrg
859
868
  return false
860
869
  end
861
870
  end
871
+
862
872
  def revoke_download(tar_files, extracted_files)
863
873
  begin
864
874
 
@@ -872,32 +882,35 @@ module Cnvrg
872
882
  return true
873
883
 
874
884
  end
885
+
875
886
  def delete_commit(commit_sha1)
876
887
  response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'DELETE')
877
888
  Cnvrg::CLI.is_response_success(response, true)
878
889
  return response
879
890
  end
891
+
880
892
  def get_commit(commit_sha1)
881
893
  response = Cnvrg::API.request("#{base_resource}/commit/#{commit_sha1}", 'GET')
882
894
  Cnvrg::CLI.is_response_success(response, true)
883
895
  return response
884
896
  end
885
897
 
886
- def start_commit(new_branch,force=false,delete_commit=nil, chunks: 0, dataset: @dataset, message:nil)
898
+ def start_commit(new_branch, force = false, delete_commit = nil, chunks: 0, dataset: @dataset, message: nil)
887
899
  begin
888
900
  #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
889
901
  # create a new commit.
890
902
  idx = (force || new_branch) ? {} : dataset.get_idx
891
903
  commit = idx[:commit]
892
904
  next_commit = idx[:next_commit]
893
- response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch,force:force,
894
- username: @owner,current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
905
+ response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug, new_branch: new_branch, force: force,
906
+ username: @owner, current_commit: commit, next_commit: next_commit, total_chunks: chunks, message: message})
895
907
  Cnvrg::CLI.is_response_success(response, true)
896
908
  return response
897
909
  rescue => e
898
910
  return false
899
911
  end
900
912
  end
913
+
901
914
  def last_valid_commit()
902
915
  begin
903
916
  #if we are pushing with force or to branch we dont need to send current/next commit cause we want to
@@ -910,9 +923,9 @@ module Cnvrg
910
923
  end
911
924
  end
912
925
 
913
- def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
926
+ def end_commit(commit_sha1, force, success: true, uploaded_files: 0)
914
927
  begin
915
- response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
928
+ response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1, force: force, success: success, uploaded_files: uploaded_files})
916
929
  Cnvrg::CLI.is_response_success(response, true)
917
930
  return response
918
931
  rescue => e
@@ -938,13 +951,13 @@ module Cnvrg
938
951
  end
939
952
 
940
953
  def get_trees(commit: "latest")
941
- response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST',{commit: commit})
954
+ response = Cnvrg::API.request("#{@base_resource}/clone_trees", 'POST', {commit: commit})
942
955
  return nil unless Cnvrg::CLI.is_response_success(response, false)
943
956
  response['result']['files']
944
957
  end
945
958
 
946
959
  def get_clone_chunk(latest_id: nil, chunk_size: 1000, commit: 'latest')
947
- response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST',{commit: commit, chunk_size: chunk_size, latest_id: latest_id})
960
+ response = Cnvrg::API.request("#{@base_resource}/clone_chunk", 'POST', {commit: commit, chunk_size: chunk_size, latest_id: latest_id})
948
961
  unless Cnvrg::CLI.is_response_success(response, false)
949
962
  Cnvrg::Logger.log_info("#{{commit: commit, chunk_size: chunk_size, latest_id: latest_id}}")
950
963
  return nil
@@ -975,26 +988,26 @@ module Cnvrg
975
988
  end
976
989
  end
977
990
 
978
- def download_multiple_chunks(commit, chunk_size=1000, progress: nil)
991
+ def download_multiple_chunks(commit, chunk_size = 1000, progress: nil)
979
992
  begin
980
- last_chunk_size = chunk_size
981
- q = { commit: commit, chunk_size: chunk_size}
982
- overall = 0
983
- while last_chunk_size > 0
984
- response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
985
- if Cnvrg::CLI.is_response_success(response, false)
986
- files = response['files']
987
- data_home = "#{Dir.pwd}/#{response['name']}"
988
- last_chunk_size = files['keys'].length
989
- break if last_chunk_size == 0
990
- res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
991
- overall += last_chunk_size
992
- q[:latest] = files['latest']
993
- else
994
- last_chunk_size = 0
993
+ last_chunk_size = chunk_size
994
+ q = {commit: commit, chunk_size: chunk_size}
995
+ overall = 0
996
+ while last_chunk_size > 0
997
+ response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/clone", 'POST', q)
998
+ if Cnvrg::CLI.is_response_success(response, false)
999
+ files = response['files']
1000
+ data_home = "#{Dir.pwd}/#{response['name']}"
1001
+ last_chunk_size = files['keys'].length
1002
+ break if last_chunk_size == 0
1003
+ res = download_multiple_files_s3(files, data_home, read_only: false, progressbar: progress)
1004
+ overall += last_chunk_size
1005
+ q[:latest] = files['latest']
1006
+ else
1007
+ last_chunk_size = 0
1008
+ end
995
1009
  end
996
- end
997
- Cnvrg::Result.new(true, "Cloned #{overall} files!")
1010
+ Cnvrg::Result.new(true, "Cloned #{overall} files!")
998
1011
  rescue => e
999
1012
  Cnvrg::Result.new(false, "Cant download chunk", e.message, e.backtrace)
1000
1013
 
@@ -1002,42 +1015,43 @@ module Cnvrg
1002
1015
  end
1003
1016
 
1004
1017
  def generate_parallel_idx
1005
-
1018
+
1006
1019
  end
1007
1020
 
1008
- def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only:false)
1021
+ def download_multiple_files_s3(files, project_home, conflict: false, progressbar: nil, read_only: false)
1009
1022
  begin
1023
+ refresh_storage_token
1010
1024
  parallel_options = {
1011
1025
  in_threads: ParallelThreads,
1012
1026
  isolation: true
1013
1027
  }
1014
1028
  Parallel.map(files["keys"], parallel_options) do |f|
1015
1029
  begin
1016
- local_path = @dataset.local_path + '/' + f['name']
1017
- Cnvrg::Logger.log_info("Downloading #{local_path}")
1018
- progressbar.progress += 1 if progressbar.present?
1019
- if local_path.end_with? "/"
1020
- @downloader.mkdir(local_path, recursive: true)
1021
- next
1022
- end
1030
+ local_path = @dataset.local_path + '/' + f['name']
1031
+ Cnvrg::Logger.log_info("Downloading #{local_path}")
1032
+ progressbar.progress += 1 if progressbar.present?
1033
+ if local_path.end_with? "/"
1034
+ @downloader.mkdir(local_path, recursive: true)
1035
+ next
1036
+ end
1023
1037
  # blob
1024
- local_path = "#{local_path}.conflict" if conflict
1025
- storage_path = f["path"]
1026
- if File.exists? local_path
1027
- Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1028
- next
1029
- end
1030
- resp = @downloader.download(storage_path, local_path)
1031
- Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
1038
+ local_path = "#{local_path}.conflict" if conflict
1039
+ storage_path = f["path"]
1040
+ if File.exists? local_path
1041
+ Cnvrg::Logger.log_info("Trying to download #{local_path} but its already exists, skipping..")
1042
+ next
1043
+ end
1044
+ resp = @downloader.download(storage_path, local_path)
1045
+ Cnvrg::Logger.log_info("Download #{local_path} success resp: #{resp}")
1032
1046
  rescue => e
1033
1047
  Cnvrg::Logger.log_error(e)
1034
1048
  end
1035
1049
  end
1036
- return Cnvrg::Result.new(true,"Downloaded successfully")
1050
+ return Cnvrg::Result.new(true, "Downloaded successfully")
1037
1051
  rescue => e
1038
- Cnvrg::Logger.log_error(e)
1039
- return Cnvrg::Result.new(false,"Could not download some files", e.message, e.backtrace)
1040
- end
1052
+ Cnvrg::Logger.log_error(e)
1053
+ return Cnvrg::Result.new(false, "Could not download some files", e.message, e.backtrace)
1041
1054
  end
1055
+ end
1042
1056
  end
1043
1057
  end
@@ -108,7 +108,7 @@ module Cnvrg
108
108
  def exec_remote(command, commit_to_run, instance_type, image_slug,schedule,local_timestamp, grid,path_to_cmd,data, data_commit,periodic_sync,
109
109
  sync_before_terminate, max_time, ds_sync_options=0,output_dir=nil,data_query=nil,
110
110
  git_commit=nil, git_branch=nil, restart_if_stuck=nil, local_folders=nil,title=nil, datasets=nil, prerun: true, requirements: true, recurring: nil,
111
- email_notification_error: false, email_notification_success: false)
111
+ email_notification_error: false, email_notification_success: false, emails_to_notify: nil)
112
112
  response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
113
113
  commit_sha1: commit_to_run,
114
114
  instance_type: instance_type,
@@ -124,7 +124,8 @@ module Cnvrg
124
124
  restart_if_stuck:restart_if_stuck, local_folders: local_folders, title:title,
125
125
  prerun: prerun, requirements: requirements, recurring: recurring,
126
126
  email_notification_error: email_notification_error,
127
- email_notification_success: email_notification_success})
127
+ email_notification_success: email_notification_success,
128
+ emails_to_notify: emails_to_notify})
128
129
 
129
130
  return response
130
131
  end
@@ -7,7 +7,7 @@ require 'cnvrg/result'
7
7
  module Cnvrg
8
8
  class Files
9
9
  ParallelThreads = Cnvrg::Helpers.parallel_threads
10
- VALID_FILE_NAME = /[\x00\\:\*\?\"<>\|]/
10
+ VALID_FILE_NAME = /[\x00\\\*\?\"<>\|]/
11
11
  LARGE_FILE=1024*1024*5
12
12
  MULTIPART_SPLIT=10000000
13
13
 
@@ -27,8 +27,16 @@ module Cnvrg
27
27
  @custom_progess = false
28
28
  @cli = cli
29
29
  @options = options
30
+ @token_issue_time = Time.current
30
31
  end
31
32
 
33
+ def refresh_storage_token
34
+ current_time = Time.current
35
+ if current_time - @token_issue_time > 3.hours
36
+ @client = @project.get_storage_client
37
+ @token_issue_time = Time.current
38
+ end
39
+ end
32
40
 
33
41
  def self.valid_file_name?(fullpath)
34
42
  VALID_FILE_NAME.match(fullpath).blank?
@@ -97,7 +105,13 @@ module Cnvrg
97
105
  files = res['files']
98
106
 
99
107
  #upload files
108
+ token_mutex = Mutex.new
100
109
  blob_ids = Parallel.map(files.keys, in_threads: ParallelThreads) do |file|
110
+
111
+ token_mutex.synchronize {
112
+ refresh_storage_token
113
+ }
114
+
101
115
  begin
102
116
  Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
103
117
  rescue => e
@@ -721,8 +735,15 @@ module Cnvrg
721
735
  in_threads: Cnvrg::Helpers.parallel_threads,
722
736
  isolation: true
723
737
  }
738
+
739
+ token_mutex = Mutex.new
740
+
724
741
  Parallel.map(files["keys"], parallel_options) do |f|
725
742
 
743
+ token_mutex.synchronize {
744
+ refresh_storage_token
745
+ }
746
+
726
747
  file_path = f["name"]
727
748
  if file_path.end_with? "/"
728
749
  # dir
@@ -370,6 +370,13 @@ module Cnvrg
370
370
  File.open(@working_dir + "/.cnvrg/config.yml", "w+") {|f| f.write new_config.to_yaml}
371
371
  end
372
372
 
373
+ def generate_git_diff
374
+ git_diff = `git diff --name-only`
375
+ git_diff.split("\n")
376
+ rescue
377
+ []
378
+ end
379
+
373
380
  def generate_output_dir(output_dir)
374
381
  Cnvrg::Logger.log_info("Generating output dir for #{output_dir}")
375
382
  upload_list = []
@@ -387,7 +394,7 @@ module Cnvrg
387
394
  upload_list << output_dir + "/"
388
395
  end
389
396
  Cnvrg::Logger.log_info("Uploading: #{upload_list.join(", ")}")
390
- return upload_list
397
+ upload_list
391
398
  end
392
399
  def generate_output_dir_tmp(output_dir)
393
400
  upload_list = []
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.6.31'
2
+ VERSION = '1.6.38'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.6.31
4
+ version: 1.6.38
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-04-01 00:00:00.000000000 Z
13
+ date: 2020-06-23 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: bundler