cnvrg 1.10.17 → 1.10.18
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/datafiles.rb +67 -42
- data/lib/cnvrg/dataset.rb +6 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8009827a102d9ea4bda3e07d95ad52a23fe7202b2670850952fb345acac7b906
|
4
|
+
data.tar.gz: aa91b22c83120d50f0631945d06f4d878e1cea4d2f9d950f6e29fbba17305f21
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2159ce3f1e3b55390b599a72b80c1b6de559554380d5b294922ddd001cff2c944cf6614bb582f184c62be6299826f4a1860257a5426c6534a254ea195bb9910c
|
7
|
+
data.tar.gz: ccb989d604602dbcfcee6f40725ae8a45875744f222b7aa717ddfe2760483361d03a16f6e016179acf82c0c493d9a3518a548d18dbb850a7cf0fc0b99aef0120
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -316,21 +316,21 @@ module Cnvrg
|
|
316
316
|
#Cnvrg::Logger.log_method(bind: binding)
|
317
317
|
retry_count += 1
|
318
318
|
|
319
|
-
|
320
|
-
puts upload_resp
|
319
|
+
Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
|
321
320
|
|
322
|
-
if retry_count >
|
323
|
-
|
321
|
+
if retry_count > 20
|
322
|
+
puts "Failed to upload files: #{Time.current}, trying next chunk"
|
323
|
+
return nil
|
324
324
|
end
|
325
325
|
sleep 5
|
326
326
|
next
|
327
327
|
end
|
328
328
|
return upload_resp['result'].with_indifferent_access
|
329
329
|
end
|
330
|
-
|
331
330
|
end
|
332
331
|
|
333
332
|
def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
|
333
|
+
Thread.report_on_exception = false
|
334
334
|
cli = CLI.new
|
335
335
|
cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
|
336
336
|
|
@@ -342,6 +342,7 @@ module Cnvrg
|
|
342
342
|
file_queue = Queue.new
|
343
343
|
progress_queue = Queue.new
|
344
344
|
worker_threads = []
|
345
|
+
progress_threads = []
|
345
346
|
|
346
347
|
# Vars to keep track of uploaded files and directories
|
347
348
|
dirs = []
|
@@ -359,45 +360,69 @@ module Cnvrg
|
|
359
360
|
cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
|
360
361
|
Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
|
361
362
|
end
|
363
|
+
while progress_queue.size > 15000
|
364
|
+
sleep(0.1)
|
365
|
+
end
|
362
366
|
progress_queue << file
|
363
367
|
end
|
364
368
|
end
|
365
369
|
end
|
366
370
|
|
367
371
|
# init the thread that handles the file upload progress and saving them in the server
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
372
|
+
threads.times do |i|
|
373
|
+
progress_threads[i] = Thread.new do
|
374
|
+
loop do
|
375
|
+
file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
|
376
|
+
unless file.nil?
|
377
|
+
blob_ids = []
|
378
|
+
dirs_to_upload = []
|
379
|
+
|
380
|
+
progress_mutex.synchronize {
|
381
|
+
progressbar.progress += 1
|
382
|
+
uploaded_files.append(file) if file[:success]
|
383
|
+
|
384
|
+
if uploaded_files.size >= chunk_size or progressbar.finished?
|
385
|
+
blob_ids = uploaded_files.map {|f| f['bv_id']}
|
386
|
+
dirs_to_upload = dirs.clone
|
387
|
+
uploaded_files = []
|
388
|
+
dirs = []
|
389
|
+
end
|
390
|
+
}
|
391
|
+
|
392
|
+
if blob_ids.present?
|
393
|
+
refresh_storage_token
|
394
|
+
Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
|
395
|
+
|
396
|
+
|
397
|
+
retry_count = 0
|
398
|
+
loop do
|
399
|
+
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
|
400
|
+
|
401
|
+
if not (Cnvrg::CLI.is_response_success(upload_resp, false))
|
402
|
+
retry_count += 1
|
403
|
+
Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
|
404
|
+
Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
|
405
|
+
if retry_count > 20
|
406
|
+
puts "Failed to save files: #{Time.current}, trying next chunk"
|
407
|
+
break
|
408
|
+
end
|
409
|
+
sleep 5
|
410
|
+
next
|
411
|
+
end
|
412
|
+
Cnvrg::Logger.info("Chunk saved on server")
|
413
|
+
break
|
414
|
+
end
|
387
415
|
end
|
388
|
-
|
389
|
-
|
390
|
-
dirs = []
|
416
|
+
else
|
417
|
+
sleep(0.1)
|
391
418
|
end
|
392
|
-
else
|
393
|
-
sleep(0.1)
|
394
|
-
end
|
395
419
|
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
420
|
+
if progressbar.finished?
|
421
|
+
Cnvrg::Logger.info("Progress bar finished closing queues")
|
422
|
+
file_queue.close()
|
423
|
+
progress_queue.close()
|
424
|
+
Thread.exit
|
425
|
+
end
|
401
426
|
end
|
402
427
|
end
|
403
428
|
end
|
@@ -409,6 +434,7 @@ module Cnvrg
|
|
409
434
|
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
|
410
435
|
Cnvrg::Logger.info("Getting files info from server")
|
411
436
|
results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
|
437
|
+
next unless results
|
412
438
|
|
413
439
|
if results['files'].blank?
|
414
440
|
progress_mutex.synchronize { progressbar.progress += tree.keys.length }
|
@@ -431,12 +457,11 @@ module Cnvrg
|
|
431
457
|
file_queue.push tree[key].merge(files_to_upload[key])
|
432
458
|
end
|
433
459
|
end
|
434
|
-
Cnvrg::Logger.info("
|
435
|
-
|
460
|
+
Cnvrg::Logger.info("Waiting to progress and workers to finish")
|
461
|
+
progress_threads.each(&:join)
|
436
462
|
worker_threads.each(&:join)
|
437
|
-
|
463
|
+
Thread.report_on_exception = true
|
438
464
|
rescue => e
|
439
|
-
puts e
|
440
465
|
Cnvrg::Logger.log_method(bind: binding)
|
441
466
|
Cnvrg::Logger.log_error(e)
|
442
467
|
raise e
|
@@ -500,7 +525,7 @@ module Cnvrg
|
|
500
525
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
501
526
|
commit_sha1: commit_sha1, file_name: file_name,
|
502
527
|
file_size: file_size, file_content_type: content_type, sha1: sha1})
|
503
|
-
|
528
|
+
|
504
529
|
|
505
530
|
end
|
506
531
|
|
@@ -547,7 +572,7 @@ module Cnvrg
|
|
547
572
|
return false
|
548
573
|
end
|
549
574
|
rescue => e
|
550
|
-
puts e.message
|
575
|
+
#puts e.message
|
551
576
|
return false
|
552
577
|
end
|
553
578
|
end
|
@@ -792,7 +817,7 @@ module Cnvrg
|
|
792
817
|
return resp
|
793
818
|
|
794
819
|
rescue => e
|
795
|
-
puts e.message
|
820
|
+
#puts e.message
|
796
821
|
return false
|
797
822
|
|
798
823
|
end
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -557,7 +557,12 @@ module Cnvrg
|
|
557
557
|
if File.directory? file
|
558
558
|
tree[label + "/"] = nil
|
559
559
|
else
|
560
|
-
|
560
|
+
begin
|
561
|
+
sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
|
562
|
+
rescue => e
|
563
|
+
#puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
|
564
|
+
next
|
565
|
+
end
|
561
566
|
file_name = File.basename file
|
562
567
|
file_size = File.size(file).to_f
|
563
568
|
mime_type = MimeMagic.by_path(file)
|
data/lib/cnvrg/version.rb
CHANGED