cnvrg 1.10.15 → 1.10.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +7 -1
- data/lib/cnvrg/datafiles.rb +67 -42
- data/lib/cnvrg/dataset.rb +6 -1
- data/lib/cnvrg/downloader/client.rb +1 -1
- data/lib/cnvrg/downloader/clients/gcp_client.rb +2 -2
- data/lib/cnvrg/version.rb +1 -1
- metadata +6 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 443373b68531083498c571cacf504da4dc02d005ef5c749a0077b7a4efce49ac
|
4
|
+
data.tar.gz: 6b944be355bd91eccc138792fbcf117dc4fdb8b505b75a7b888dcd74f47aa1ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: b7ff78d2eceb2a420f46d2ed96ac06ab2c96cc031d8949581069f2b4a4c9d45f210120024c2c25fe93b4542c9f0de0d3dcf893745911f129b8986cda716d3505
|
7
|
+
data.tar.gz: 56d85bf1fb4d16c1ec3ce46dfd5e38363c71a5242daaddb6e7f8cfa335526961abf4c6cf2345426fd8689d97e7d9a4e03b75c34e16ca32d440aca42459ed07e3
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -3096,6 +3096,11 @@ module Cnvrg
|
|
3096
3096
|
emails = options["emails"]
|
3097
3097
|
wait = options["wait"]
|
3098
3098
|
|
3099
|
+
if wait && local
|
3100
|
+
log_message("WARN: `wait` option is not valid for local experiment, ignoring it", Thor::Shell::Color::YELLOW)
|
3101
|
+
wait = false
|
3102
|
+
end
|
3103
|
+
|
3099
3104
|
if !data.present? and data_query.present?
|
3100
3105
|
log_message("Please provide data with data_query", Thor::Shell::Color::RED)
|
3101
3106
|
exit(1)
|
@@ -3637,7 +3642,8 @@ module Cnvrg
|
|
3637
3642
|
sleep 3
|
3638
3643
|
else
|
3639
3644
|
result.each do |slug, value|
|
3640
|
-
|
3645
|
+
exit_status = value['exit_status']
|
3646
|
+
puts "Experiment #{slug} was exited with status #{exit_status}"
|
3641
3647
|
end
|
3642
3648
|
break
|
3643
3649
|
end
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -316,21 +316,21 @@ module Cnvrg
|
|
316
316
|
#Cnvrg::Logger.log_method(bind: binding)
|
317
317
|
retry_count += 1
|
318
318
|
|
319
|
-
|
320
|
-
puts upload_resp
|
319
|
+
Cnvrg::Logger.log_info("Failed request upload files: #{Time.current}, retry: #{retry_count}")
|
321
320
|
|
322
|
-
if retry_count >
|
323
|
-
|
321
|
+
if retry_count > 20
|
322
|
+
puts "Failed to upload files: #{Time.current}, trying next chunk"
|
323
|
+
return nil
|
324
324
|
end
|
325
325
|
sleep 5
|
326
326
|
next
|
327
327
|
end
|
328
328
|
return upload_resp['result'].with_indifferent_access
|
329
329
|
end
|
330
|
-
|
331
330
|
end
|
332
331
|
|
333
332
|
def upload_multiple_files_optimized(files, commit_sha1, threads: 15, chunk_size: 1000, override: false, new_branch: false, prefix: '', partial_commit: nil)
|
333
|
+
Thread.report_on_exception = false
|
334
334
|
cli = CLI.new
|
335
335
|
cli.log_message("Using #{threads} threads with chunk size of #{chunk_size}.", Thor::Shell::Color::GREEN)
|
336
336
|
|
@@ -342,6 +342,7 @@ module Cnvrg
|
|
342
342
|
file_queue = Queue.new
|
343
343
|
progress_queue = Queue.new
|
344
344
|
worker_threads = []
|
345
|
+
progress_threads = []
|
345
346
|
|
346
347
|
# Vars to keep track of uploaded files and directories
|
347
348
|
dirs = []
|
@@ -359,45 +360,69 @@ module Cnvrg
|
|
359
360
|
cli.log_message("Error while uploading file: #{file[:absolute_path]}", Thor::Shell::Color::RED)
|
360
361
|
Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
|
361
362
|
end
|
363
|
+
while progress_queue.size > 15000
|
364
|
+
sleep(0.1)
|
365
|
+
end
|
362
366
|
progress_queue << file
|
363
367
|
end
|
364
368
|
end
|
365
369
|
end
|
366
370
|
|
367
371
|
# init the thread that handles the file upload progress and saving them in the server
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
372
|
+
threads.times do |i|
|
373
|
+
progress_threads[i] = Thread.new do
|
374
|
+
loop do
|
375
|
+
file = progress_queue.deq(non_block: true) rescue nil # to prevent deadlocks
|
376
|
+
unless file.nil?
|
377
|
+
blob_ids = []
|
378
|
+
dirs_to_upload = []
|
379
|
+
|
380
|
+
progress_mutex.synchronize {
|
381
|
+
progressbar.progress += 1
|
382
|
+
uploaded_files.append(file) if file[:success]
|
383
|
+
|
384
|
+
if uploaded_files.size >= chunk_size or progressbar.finished?
|
385
|
+
blob_ids = uploaded_files.map {|f| f['bv_id']}
|
386
|
+
dirs_to_upload = dirs.clone
|
387
|
+
uploaded_files = []
|
388
|
+
dirs = []
|
389
|
+
end
|
390
|
+
}
|
391
|
+
|
392
|
+
if blob_ids.present?
|
393
|
+
refresh_storage_token
|
394
|
+
Cnvrg::Logger.info("Finished upload chunk of #{chunk_size} files, Sending Upload files save")
|
395
|
+
|
396
|
+
|
397
|
+
retry_count = 0
|
398
|
+
loop do
|
399
|
+
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs_to_upload})
|
400
|
+
|
401
|
+
if not (Cnvrg::CLI.is_response_success(upload_resp, false))
|
402
|
+
retry_count += 1
|
403
|
+
Cnvrg::Logger.log_error_message("Failed request save files: #{Time.current}, retry: #{retry_count}")
|
404
|
+
Cnvrg::Logger.info("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
|
405
|
+
if retry_count > 20
|
406
|
+
puts "Failed to save files: #{Time.current}, trying next chunk"
|
407
|
+
break
|
408
|
+
end
|
409
|
+
sleep 5
|
410
|
+
next
|
411
|
+
end
|
412
|
+
Cnvrg::Logger.info("Chunk saved on server")
|
413
|
+
break
|
414
|
+
end
|
387
415
|
end
|
388
|
-
|
389
|
-
|
390
|
-
dirs = []
|
416
|
+
else
|
417
|
+
sleep(0.1)
|
391
418
|
end
|
392
|
-
else
|
393
|
-
sleep(0.1)
|
394
|
-
end
|
395
419
|
|
396
|
-
|
397
|
-
|
398
|
-
|
399
|
-
|
400
|
-
|
420
|
+
if progressbar.finished?
|
421
|
+
Cnvrg::Logger.info("Progress bar finished closing queues")
|
422
|
+
file_queue.close()
|
423
|
+
progress_queue.close()
|
424
|
+
Thread.exit
|
425
|
+
end
|
401
426
|
end
|
402
427
|
end
|
403
428
|
end
|
@@ -409,6 +434,7 @@ module Cnvrg
|
|
409
434
|
tree = @dataset.generate_chunked_idx(files_chunk, prefix: prefix, threads: threads)
|
410
435
|
Cnvrg::Logger.info("Getting files info from server")
|
411
436
|
results = request_upload_files(commit_sha1, tree, override, new_branch, partial_commit)
|
437
|
+
next unless results
|
412
438
|
|
413
439
|
if results['files'].blank?
|
414
440
|
progress_mutex.synchronize { progressbar.progress += tree.keys.length }
|
@@ -431,12 +457,11 @@ module Cnvrg
|
|
431
457
|
file_queue.push tree[key].merge(files_to_upload[key])
|
432
458
|
end
|
433
459
|
end
|
434
|
-
Cnvrg::Logger.info("
|
435
|
-
|
460
|
+
Cnvrg::Logger.info("Waiting to progress and workers to finish")
|
461
|
+
progress_threads.each(&:join)
|
436
462
|
worker_threads.each(&:join)
|
437
|
-
|
463
|
+
Thread.report_on_exception = true
|
438
464
|
rescue => e
|
439
|
-
puts e
|
440
465
|
Cnvrg::Logger.log_method(bind: binding)
|
441
466
|
Cnvrg::Logger.log_error(e)
|
442
467
|
raise e
|
@@ -500,7 +525,7 @@ module Cnvrg
|
|
500
525
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
501
526
|
commit_sha1: commit_sha1, file_name: file_name,
|
502
527
|
file_size: file_size, file_content_type: content_type, sha1: sha1})
|
503
|
-
|
528
|
+
|
504
529
|
|
505
530
|
end
|
506
531
|
|
@@ -547,7 +572,7 @@ module Cnvrg
|
|
547
572
|
return false
|
548
573
|
end
|
549
574
|
rescue => e
|
550
|
-
puts e.message
|
575
|
+
#puts e.message
|
551
576
|
return false
|
552
577
|
end
|
553
578
|
end
|
@@ -792,7 +817,7 @@ module Cnvrg
|
|
792
817
|
return resp
|
793
818
|
|
794
819
|
rescue => e
|
795
|
-
puts e.message
|
820
|
+
#puts e.message
|
796
821
|
return false
|
797
822
|
|
798
823
|
end
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -557,7 +557,12 @@ module Cnvrg
|
|
557
557
|
if File.directory? file
|
558
558
|
tree[label + "/"] = nil
|
559
559
|
else
|
560
|
-
|
560
|
+
begin
|
561
|
+
sha1 = OpenSSL::Digest::SHA1.file(file).hexdigest
|
562
|
+
rescue => e
|
563
|
+
#puts "Could'nt calculate sha1 for: #{file}, Error: #{e.message}"
|
564
|
+
next
|
565
|
+
end
|
561
566
|
file_name = File.basename file
|
562
567
|
file_size = File.size(file).to_f
|
563
568
|
mime_type = MimeMagic.by_path(file)
|
@@ -11,7 +11,7 @@ module Cnvrg
|
|
11
11
|
@tempfile = nil
|
12
12
|
@bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
|
13
13
|
init_gcp_credentials
|
14
|
-
@storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries:
|
14
|
+
@storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 50, timeout: 43200)
|
15
15
|
@bucket = @storage.bucket(@bucket_name)
|
16
16
|
@bucket.name
|
17
17
|
rescue => e
|
@@ -47,4 +47,4 @@ module Cnvrg
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
end
|
50
|
-
end
|
50
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,16 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.10.
|
4
|
+
version: 1.10.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
8
8
|
- Leah Kolben
|
9
9
|
- Omer Shacham
|
10
|
-
autorequire:
|
10
|
+
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-09-
|
13
|
+
date: 2020-09-22 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -438,7 +438,7 @@ files:
|
|
438
438
|
homepage: https://cnvrg.io
|
439
439
|
licenses: []
|
440
440
|
metadata: {}
|
441
|
-
post_install_message:
|
441
|
+
post_install_message:
|
442
442
|
rdoc_options: []
|
443
443
|
require_paths:
|
444
444
|
- lib
|
@@ -453,8 +453,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
453
453
|
- !ruby/object:Gem::Version
|
454
454
|
version: '0'
|
455
455
|
requirements: []
|
456
|
-
rubygems_version: 3.
|
457
|
-
signing_key:
|
456
|
+
rubygems_version: 3.1.2
|
457
|
+
signing_key:
|
458
458
|
specification_version: 4
|
459
459
|
summary: A CLI tool for interacting with cnvrg.io.
|
460
460
|
test_files: []
|