cnvrg 1.10.17 → 1.10.18

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 29b4ab96b94066d5e799cbd5e648eaae0871a634ac6383a66ca3c82a3af9a038
4
- data.tar.gz: aca3f52a3959cf6db6614bed5b7669c41e9607f0abf25ec83e870783c997f484
3
+ metadata.gz: 8009827a102d9ea4bda3e07d95ad52a23fe7202b2670850952fb345acac7b906
4
+ data.tar.gz: aa91b22c83120d50f0631945d06f4d878e1cea4d2f9d950f6e29fbba17305f21
5
5
  SHA512:
6
- metadata.gz: 96fbd5de39c5aa44d93e24c529b5e149cbd8f689c56329e7c5b4a6f9990d19ca8a7d1fdb1f5f00b8abc05c8dccc061a74de65ac8c81524c27f65b0e349dfdc23
7
- data.tar.gz: 2412c51b6926725be234c022faca6191110caf45de2c83381b171e411f7d7bee78e74d519b91f42e01d07b69413df5df009747f35bd34e480d5733718ccd824c
6
+ metadata.gz: 2159ce3f1e3b55390b599a72b80c1b6de559554380d5b294922ddd001cff2c944cf6614bb582f184c62be6299826f4a1860257a5426c6534a254ea195bb9910c
7
+ data.tar.gz: ccb989d604602dbcfcee6f40725ae8a45875744f222b7aa717ddfe2760483361d03a16f6e016179acf82c0c493d9a3518a548d18dbb850a7cf0fc0b99aef0120
@@ -316,21 +316,21 @@ module Cnvrg
316
316
  #Cnvrg::Logger.log_method(bind: binding)
317
317
  retry_count += 1
318
318
 
319
- puts "Failed request upload files: #{Time.current}"
320
- puts upload_resp
319
+ Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
321
320
 
322
- if retry_count > 5
323
- raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
321
+ if retry_count > 20
322
+ puts "Failed to upload files: #{Time.current}, trying next chunk"
323
+ return nil
324
324
  end
325
325
  sleep 5
326
326
  next
327
327
  end
328
328
  return upload_resp['result'].with_indifferent_access
329
329
  end
330
-
331
330
  end
332
331
 
333
332
  def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
333
+ Thread.report_on_exception = false
334
334
  cli = CLI.new
335
335
  cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
336
336
 
@@ -342,6 +342,7 @@ module Cnvrg
342
342
  file_queue = Queue.new
343
343
  progress_queue = Queue.new
344
344
  worker_threads = []
345
+ progress_threads = []
345
346
 
346
347
  # Vars to keep track of uploaded files and directories
347
348
  dirs = []
@@ -359,45 +360,69 @@ module Cnvrg
359
360
  cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
360
361
  Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
361
362
  end
363
+ while progress_queue.size > 15000
364
+ sleep(0.1)
365
+ end
362
366
  progress_queue << file
363
367
  end
364
368
  end
365
369
  end
366
370
 
367
371
  # init the thread that handles the file upload progress and saving them in the server
368
- progress_thread = Thread.new do
369
- loop do
370
- file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
371
- unless file.nil?
372
- progress_mutex.synchronize {
373
- progressbar.progress += 1
374
- uploaded_files.append(file) if file[:success]
375
- }
376
-
377
- if uploaded_files.size == chunk_size or progressbar.finished?
378
- refresh_storage_token
379
- Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files")
380
- Cnvrg::Logger.info("Sending Upload files save")
381
- blob_ids = uploaded_files.map {|f| f['bv_id']}
382
- Cnvrg::Logger.info("Sending chunk to server")
383
- upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
384
- unless Cnvrg::CLI.is_response_success(upload_resp, false)
385
- Cnvrg::Logger.log_method(bind: binding)
386
- raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
372
+ threads.times do |i|
373
+ progress_threads[i] = Thread.new do
374
+ loop do
375
+ file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
376
+ unless file.nil?
377
+ blob_ids = []
378
+ dirs_to_upload = []
379
+
380
+ progress_mutex.synchronize {
381
+ progressbar.progress += 1
382
+ uploaded_files.append(file) if file[:success]
383
+
384
+ if uploaded_files.size >= chunk_size or progressbar.finished?
385
+ blob_ids = uploaded_files.map {|f| f['bv_id']}
386
+ dirs_to_upload = dirs.clone
387
+ uploaded_files = []
388
+ dirs = []
389
+ end
390
+ }
391
+
392
+ if blob_ids.present?
393
+ refresh_storage_token
394
+ Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
395
+
396
+
397
+ retry_count = 0
398
+ loop do
399
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
400
+
401
+ if not (Cnvrg::CLI.is_response_success(upload_resp, false))
402
+ retry_count += 1
403
+ Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
404
+ Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
405
+ if retry_count > 20
406
+ puts "Failed to save files: #{Time.current}, trying next chunk"
407
+ break
408
+ end
409
+ sleep 5
410
+ next
411
+ end
412
+ Cnvrg::Logger.info("Chunk saved on server")
413
+ break
414
+ end
387
415
  end
388
- Cnvrg::Logger.info("Chunk saved on server")
389
- uploaded_files = []
390
- dirs = []
416
+ else
417
+ sleep(0.1)
391
418
  end
392
- else
393
- sleep(0.1)
394
- end
395
419
 
396
- if progressbar.finished?
397
- Cnvrg::Logger.info("Progress bar finished closing queues")
398
- file_queue.close()
399
- progress_queue.close()
400
- Thread.exit
420
+ if progressbar.finished?
421
+ Cnvrg::Logger.info("Progress bar finished closing queues")
422
+ file_queue.close()
423
+ progress_queue.close()
424
+ Thread.exit
425
+ end
401
426
  end
402
427
  end
403
428
  end
@@ -409,6 +434,7 @@ module Cnvrg
409
434
  tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
410
435
  Cnvrg::Logger.info("Getting files info from server")
411
436
  results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
437
+ next unless results
412
438
 
413
439
  if results['files'].blank?
414
440
  progress_mutex.synchronize { progressbar.progress += tree.keys.length }
@@ -431,12 +457,11 @@ module Cnvrg
431
457
  file_queue.push tree[key].merge(files_to_upload[key])
432
458
  end
433
459
  end
434
- Cnvrg::Logger.info("Finishing sub processes of datasets' upload")
435
- progress_thread.join()
460
+ Cnvrg::Logger.info("Waiting to progress and workers to finish")
461
+ progress_threads.each(&:join)
436
462
  worker_threads.each(&:join)
437
-
463
+ Thread.report_on_exception = true
438
464
  rescue => e
439
- puts e
440
465
  Cnvrg::Logger.log_method(bind: binding)
441
466
  Cnvrg::Logger.log_error(e)
442
467
  raise e
@@ -500,7 +525,7 @@ module Cnvrg
500
525
  upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
501
526
  commit_sha1: commit_sha1, file_name: file_name,
502
527
  file_size: file_size, file_content_type: content_type, sha1: sha1})
503
- puts upload_resp
528
+
504
529
 
505
530
  end
506
531
 
@@ -547,7 +572,7 @@ module Cnvrg
547
572
  return false
548
573
  end
549
574
  rescue => e
550
- puts e.message
575
+ #puts e.message
551
576
  return false
552
577
  end
553
578
  end
@@ -792,7 +817,7 @@ module Cnvrg
792
817
  return resp
793
818
 
794
819
  rescue => e
795
- puts e.message
820
+ #puts e.message
796
821
  return false
797
822
 
798
823
  end
@@ -557,7 +557,12 @@ module Cnvrg
557
557
  if File.directory? file
558
558
  tree[label + "/"] = nil
559
559
  else
560
- sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
560
+ begin
561
+ sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
562
+ rescue => e
563
+ #puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
564
+ next
565
+ end
561
566
  file_name = File.basename file
562
567
  file_size = File.size(file).to_f
563
568
  mime_type = MimeMagic.by_path(file)
@@ -1,3 +1,3 @@
1
1
  module Cnvrg
2
- VERSION = '1.10.17'
2
+ VERSION = '1.10.18'
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: cnvrg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.17
4
+ version: 1.10.18
5
5
  platform: ruby
6
6
  authors:
7
7
  - Yochay Ettun