cnvrg 0.6.3 → 0.6.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +56 -58
- data/lib/cnvrg/datafiles.rb +44 -7
- data/lib/cnvrg/dataset.rb +16 -7
- data/lib/cnvrg/helpers.rb +2 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88c66150eee5e836d00d2cbf4983ab978b227f6f
|
4
|
+
data.tar.gz: 28ff50dfd3e535e640ebd5d7a2333dc55f954176
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d5276cacdecc9b1860e1f1a966d93186b9c5ff00320f53d65cf4bed39ac67ff4ba927a73d232cd98fd3d54a3deb237dde92289079d648583bf30a7672ffdf975
|
7
|
+
data.tar.gz: 9c84dca4965fc101de56e573fbd655e290c34c068c9d0598b524a82436ddb332af02228407e8c7c517734a992ad67ba489b64d2d35092deb248ce59cdff03455
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
|
-
require "pty"
|
2
|
+
require "pty" unless !!(RUBY_PLATFORM =~ /mswin32|mingw32/)
|
3
|
+
|
3
4
|
require 'etc'
|
4
5
|
require 'parallel'
|
5
6
|
require 'netrc'
|
@@ -844,6 +845,7 @@ module Cnvrg
|
|
844
845
|
:starting_at => 0,
|
845
846
|
:total => files_count,
|
846
847
|
:autofinish => true)
|
848
|
+
|
847
849
|
while files['keys'].length > 0
|
848
850
|
@files.download_multiple_files_s3(files, dataset_home, progressbar: progressbar)
|
849
851
|
downloaded_files += files['keys'].length
|
@@ -1922,8 +1924,10 @@ module Cnvrg
|
|
1922
1924
|
log_start(__method__, args, options)
|
1923
1925
|
log_message('Syncing dataset', Thor::Shell::Color::BLUE, !options["verbose"])
|
1924
1926
|
if !force and !init
|
1925
|
-
|
1927
|
+
# w(verbose=false, new_branch=false,sync=false, commit=nil,all_files=true)
|
1928
|
+
invoke :download_data_new,[verbose, new_branch, true, commit, all_files], :new_branch=>new_branch, :direct=>false, :force =>force
|
1926
1929
|
end
|
1930
|
+
# w(new_branch, verbose,sync,force, tags, chunk_size)
|
1927
1931
|
invoke :upload_data_new,[new_branch, verbose,true,force, tags, chunk_size], :new_branch=>new_branch,
|
1928
1932
|
:direct=>false, :force =>force, :sync =>true, :tags =>tags, :parallel => parallel
|
1929
1933
|
|
@@ -1936,6 +1940,7 @@ module Cnvrg
|
|
1936
1940
|
method_option :tags, :type => :boolean, :aliases => ["--tags"], :desc => "upload file tags", :default => false
|
1937
1941
|
method_option :parallel, :type => :numeric, :aliases => ["-p", "--parallel"], :desc => "uparallel upload at the same time", :default => 15
|
1938
1942
|
def upload_data_new(new_branch, verbose,sync,force, tags, chunk_size)
|
1943
|
+
begin
|
1939
1944
|
commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
|
1940
1945
|
upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
|
1941
1946
|
if upload_res < 0
|
@@ -1962,11 +1967,21 @@ module Cnvrg
|
|
1962
1967
|
log_message('There was some error in uploading Tags', Thor::Shell::Color::RED)
|
1963
1968
|
end
|
1964
1969
|
end
|
1965
|
-
|
1966
|
-
|
1967
|
-
|
1968
|
-
|
1969
|
-
|
1970
|
+
rescue => e
|
1971
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
1972
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1973
|
+
return false if dataset_dir.blank?
|
1974
|
+
@dataset = Dataset.new(dataset_dir)
|
1975
|
+
@dataset.revert_to_last_commit
|
1976
|
+
return false
|
1977
|
+
rescue SignalException
|
1978
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
1979
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1980
|
+
return false if dataset_dir.blank?
|
1981
|
+
@dataset = Dataset.new(dataset_dir)
|
1982
|
+
@dataset.revert_to_last_commit
|
1983
|
+
return false
|
1984
|
+
end
|
1970
1985
|
end
|
1971
1986
|
|
1972
1987
|
desc 'start_commit', 'start data commit'
|
@@ -2014,18 +2029,7 @@ module Cnvrg
|
|
2014
2029
|
puts commit_sha1
|
2015
2030
|
end
|
2016
2031
|
return commit_sha1
|
2017
|
-
rescue => e
|
2018
|
-
if commit_sha1.present?
|
2019
|
-
@dataset.set_next_commit(commit_sha1)
|
2020
|
-
end
|
2021
|
-
rescue SignalException
|
2022
|
-
log_message("Aborting..", Thor::Shell::Color::YELLOW)
|
2023
|
-
if commit_sha1.present?
|
2024
|
-
@dataset.set_next_commit(commit_sha1)
|
2025
|
-
end
|
2026
|
-
exit(1)
|
2027
2032
|
end
|
2028
|
-
|
2029
2033
|
end
|
2030
2034
|
desc 'end_commit', 'start data commit'
|
2031
2035
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
@@ -2049,18 +2053,9 @@ module Cnvrg
|
|
2049
2053
|
log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
|
2050
2054
|
@dataset.set_next_commit(resp["result"]["commit_sha1"]) # no need fo rboth methods FIX
|
2051
2055
|
@dataset.remove_next_commit()
|
2052
|
-
|
2053
2056
|
end
|
2054
|
-
|
2055
2057
|
end
|
2056
|
-
rescue => e
|
2057
|
-
puts e
|
2058
|
-
rescue SignalException
|
2059
|
-
@dataset.set_next_commit(commit)
|
2060
|
-
log_message("Aborting", Thor::Shell::Color::YELLOW)
|
2061
|
-
exit(1)
|
2062
2058
|
end
|
2063
|
-
|
2064
2059
|
end
|
2065
2060
|
desc 'list_files', 'list files in dataset'
|
2066
2061
|
method_option :json, :type => :boolean, :aliases => ["-j","--json"],:default => true, :desc => "response as json"
|
@@ -2137,12 +2132,13 @@ module Cnvrg
|
|
2137
2132
|
files_uploaded = 0
|
2138
2133
|
all_files.each_slice(chunk_size).each do |list_files|
|
2139
2134
|
temp_tree = @dataset.generate_chunked_idx(list_files, threads: parallel_threads)
|
2135
|
+
|
2140
2136
|
upload_resp = @files.upload_multiple_files(new_commit, temp_tree, threads: parallel_threads, force: force, new_branch: new_branch, progressbar: progressbar)
|
2141
2137
|
|
2142
2138
|
if upload_resp.is_a? Cnvrg::Result
|
2143
2139
|
log_message(upload_resp.msg, upload_resp.msg_color)
|
2144
|
-
|
2145
|
-
return -1
|
2140
|
+
raise SignalException.new(upload_resp.msg)
|
2141
|
+
# return -1
|
2146
2142
|
|
2147
2143
|
else
|
2148
2144
|
files_uploaded += upload_resp
|
@@ -2155,9 +2151,6 @@ module Cnvrg
|
|
2155
2151
|
@dataset.write_idx(new_tree, new_commit)
|
2156
2152
|
end
|
2157
2153
|
return files_uploaded
|
2158
|
-
rescue Exception => e
|
2159
|
-
puts e
|
2160
|
-
return false
|
2161
2154
|
end
|
2162
2155
|
|
2163
2156
|
# local_idx = @dataset.generate_idx
|
@@ -2522,7 +2515,6 @@ module Cnvrg
|
|
2522
2515
|
rescue SignalException
|
2523
2516
|
log_message("Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2524
2517
|
@files.rollback_commit(commit_sha1)
|
2525
|
-
|
2526
2518
|
return false
|
2527
2519
|
end
|
2528
2520
|
end
|
@@ -2767,30 +2759,30 @@ module Cnvrg
|
|
2767
2759
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2768
2760
|
@dataset = Dataset.new(dataset_dir)
|
2769
2761
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2770
|
-
commit_to_download = commit
|
2771
|
-
local_idx = @dataset.generate_idx
|
2772
2762
|
all_files = all_files
|
2773
|
-
res = @dataset.compare_idx_download(
|
2763
|
+
res = @dataset.compare_idx_download(all_files: all_files, desired_commit: commit)
|
2764
|
+
unless CLI.is_response_success(res, false)
|
2765
|
+
log_message("Cant find the desired commit, please check it or try to download without it.", Thor::Shell::Color::RED)
|
2766
|
+
exit(1)
|
2767
|
+
end
|
2774
2768
|
result = res["result"]["tree"]
|
2775
2769
|
commit = res["result"]["commit"]
|
2776
|
-
|
2777
|
-
|
2778
|
-
|
2779
|
-
|
2780
|
-
update_total = result["updated_on_server"].size + result["conflicts"].size + result["deleted"].size
|
2781
|
-
successful_changes = 0
|
2782
|
-
if update_total ==1
|
2783
|
-
log_message("Downloading #{update_total} file", Thor::Shell::Color::BLUE, !sync)
|
2784
|
-
elsif update_total == 0
|
2785
|
-
log_message("Dataset is up to date", Thor::Shell::Color::GREEN, !sync)
|
2770
|
+
update_total = [result['added'], result["updated_on_server"], result["conflicts"], result["deleted"]].compact.flatten.size
|
2771
|
+
successful_changes = 0
|
2772
|
+
if update_total == 0
|
2773
|
+
log_message("Dataset is up to date", Thor::Shell::Color::GREEN) if !sync
|
2786
2774
|
return true
|
2787
2775
|
elsif options["verbose"]
|
2788
2776
|
log_message("Downloading #{update_total} files", Thor::Shell::Color::BLUE)
|
2789
2777
|
else
|
2790
2778
|
log_message("Syncing Dataset", Thor::Shell::Color::BLUE, !sync)
|
2791
2779
|
end
|
2780
|
+
conflicts = @files.mark_conflicts(result)
|
2781
|
+
|
2782
|
+
log_message("Found some conflicts, check .conflict files.", Thor::Shell::Color::BLUE) if conflicts > 0
|
2792
2783
|
update_res = @files.download_files_in_chunks(result["updated_on_server"]) if result["updated_on_server"].present?
|
2793
|
-
|
2784
|
+
added_res = @files.download_files_in_chunks(result["added"]) if result["added"].present?
|
2785
|
+
# conflict_res = @files.download_files_in_chunks(result["conflicts"], conflict: true) if result["conflicts"].present?
|
2794
2786
|
deleted = result["deleted"].to_a
|
2795
2787
|
delete_res = @files.delete_commit_files_local(deleted)
|
2796
2788
|
if !delete_res
|
@@ -2798,7 +2790,10 @@ module Cnvrg
|
|
2798
2790
|
log_message("Couldn't download, Rolling Back all changes.", Thor::Shell::Color::RED)
|
2799
2791
|
exit(1)
|
2800
2792
|
end
|
2801
|
-
|
2793
|
+
success = (update_res.blank? or update_res.is_success?)
|
2794
|
+
success &= (delete_res.blank? or delete_res.is_success?)
|
2795
|
+
success &= (added_res.blank? or added_res.is_success?)
|
2796
|
+
if success
|
2802
2797
|
# update idx with latest commit
|
2803
2798
|
@dataset.update_idx_with_commit!(commit)
|
2804
2799
|
check = Helpers.checkmark()
|
@@ -2809,17 +2804,15 @@ module Cnvrg
|
|
2809
2804
|
else
|
2810
2805
|
log_message("#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN, ((sync or options["sync"]) ? false : true))
|
2811
2806
|
end
|
2812
|
-
return true
|
2813
|
-
else
|
2814
|
-
@dataset.update_idx_with_commit!(commit_to_download)
|
2807
|
+
return true
|
2815
2808
|
end
|
2809
|
+
rescue SignalException
|
2810
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
2811
|
+
exit(1)
|
2816
2812
|
rescue => e
|
2817
2813
|
log_message("Error occurred, \nAborting", Thor::Shell::Color::BLUE)
|
2818
2814
|
log_error(e)
|
2819
2815
|
exit(1)
|
2820
|
-
rescue SignalException
|
2821
|
-
say "\nAborting", Thor::Shell::Color::BLUE
|
2822
|
-
exit(1)
|
2823
2816
|
end
|
2824
2817
|
end
|
2825
2818
|
|
@@ -3339,10 +3332,15 @@ module Cnvrg
|
|
3339
3332
|
options_hash = Hash[options]
|
3340
3333
|
|
3341
3334
|
if local
|
3342
|
-
|
3343
|
-
|
3344
|
-
|
3345
|
-
|
3335
|
+
if Cnvrg::Helpers.windows?
|
3336
|
+
say "Windows is currently not supported for running experiments locally"
|
3337
|
+
return
|
3338
|
+
else
|
3339
|
+
invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
|
3340
|
+
:log => log, :email_notification => email_notification, :upload_output => upload_output,
|
3341
|
+
:commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore, :force => force, :output_dir=>output_dir, :data_query=>data_query
|
3342
|
+
return
|
3343
|
+
end
|
3346
3344
|
else
|
3347
3345
|
if !periodic_sync.nil? and !periodic_sync.empty?
|
3348
3346
|
if /^\d{2}$/ === periodic_sync
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -20,6 +20,39 @@ module Cnvrg
|
|
20
20
|
@base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
|
21
21
|
end
|
22
22
|
|
23
|
+
def check_file_sha1(filename, org_sha1, tag: 'conflict')
|
24
|
+
file_loc = "#{Dir.pwd}/#{filename}"
|
25
|
+
sha1 = OpenSSL::Digest::SHA1.file(file_loc).hexdigest
|
26
|
+
return 0 if sha1 == org_sha1
|
27
|
+
FileUtils.cp(file_loc, "#{file_loc}.#{tag}")
|
28
|
+
return 1
|
29
|
+
end
|
30
|
+
|
31
|
+
def check_files_sha1(files, resolver, tag)
|
32
|
+
conflicts = 0
|
33
|
+
files.each do |file|
|
34
|
+
next if file.ends_with? '/'
|
35
|
+
sha1 = resolver.fetch(file, {}).fetch("sha1", nil)
|
36
|
+
conflicts += self.check_file_sha1(file, sha1, tag: tag)
|
37
|
+
end
|
38
|
+
conflicts
|
39
|
+
end
|
40
|
+
|
41
|
+
def mark_conflicts(results)
|
42
|
+
begin
|
43
|
+
updated = results["updated_on_server"]
|
44
|
+
deleted = results["deleted"]
|
45
|
+
resolver = results['resolved_files']
|
46
|
+
overall = 0
|
47
|
+
overall += self.check_files_sha1(updated, resolver, "conflict")
|
48
|
+
overall += self.check_files_sha1(deleted, resolver, "deleted")
|
49
|
+
overall
|
50
|
+
rescue => e
|
51
|
+
0
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
|
23
56
|
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil)
|
24
57
|
random_file_name = (0...8).map { (65 + rand(26)).chr }.join #needed to create a file for post_file..
|
25
58
|
# each file have: {sha1: sha1, file_name: file_name, file_size: file_size, content_type: content_type, absolute_path, relative_path}
|
@@ -29,9 +62,9 @@ module Cnvrg
|
|
29
62
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
|
30
63
|
return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
31
64
|
results = upload_resp['result'].with_indifferent_access
|
32
|
-
|
33
|
-
|
34
|
-
|
65
|
+
if results['files'].blank?
|
66
|
+
return 0
|
67
|
+
end
|
35
68
|
props = Cnvrg::Helpers.get_s3_props(results)
|
36
69
|
client = props[:client]
|
37
70
|
bucket = props[:bucket]
|
@@ -45,7 +78,7 @@ module Cnvrg
|
|
45
78
|
progressbar.progress += 1 if progressbar.present?
|
46
79
|
end
|
47
80
|
blob_ids = files.values.map{|f| f['bv_id']}
|
48
|
-
dirs = tree.keys.select{|k| tree[k].nil? }
|
81
|
+
dirs = tree.keys.select{|k| tree[k].nil? } || []
|
49
82
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
|
50
83
|
return files.keys.length
|
51
84
|
rescue => e
|
@@ -808,15 +841,16 @@ module Cnvrg
|
|
808
841
|
(1..5).each do |i|
|
809
842
|
response = Cnvrg::API.request("users/#{@owner}/datasets/#{@dataset_slug}/download_multi", 'POST', {files: files})
|
810
843
|
next unless Cnvrg::CLI.is_response_success(response, false) #trying to api request 5 times.
|
811
|
-
|
844
|
+
files_to_download = response['files']
|
812
845
|
data_home = "#{Dir.pwd}/#{response['name']}"
|
813
|
-
res = download_multiple_files_s3(
|
846
|
+
res = download_multiple_files_s3(files_to_download, data_home, conflict: conflict)
|
814
847
|
next unless res.is_success? #try again..
|
815
|
-
return
|
848
|
+
return files_to_download['keys'].length
|
816
849
|
end
|
817
850
|
end
|
818
851
|
|
819
852
|
def download_multiple_chunks(commit, chunk_size=1000)
|
853
|
+
begin
|
820
854
|
last_chunk_size = chunk_size
|
821
855
|
q = { commit: commit, chunk_size: chunk_size}
|
822
856
|
overall = 0
|
@@ -835,6 +869,9 @@ module Cnvrg
|
|
835
869
|
end
|
836
870
|
end
|
837
871
|
Cnvrg::Result.new(true, "Cloned #{overall} files!")
|
872
|
+
rescue => e
|
873
|
+
Cnvrg::Result.new(false, "Cant download chunk")
|
874
|
+
end
|
838
875
|
end
|
839
876
|
|
840
877
|
def generate_parallel_idx
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -314,6 +314,17 @@ module Cnvrg
|
|
314
314
|
return tree
|
315
315
|
end
|
316
316
|
|
317
|
+
def revert_to_last_commit(commit: nil)
|
318
|
+
if commit.blank?
|
319
|
+
resp = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/last_valid_commit", 'GET')
|
320
|
+
if CLI.is_response_success(resp, false)
|
321
|
+
commit = resp['result']['commit_sha1']
|
322
|
+
end
|
323
|
+
end
|
324
|
+
self.update_idx_with_commit(commit) if commit.present?
|
325
|
+
self.revert_next_commit
|
326
|
+
end
|
327
|
+
|
317
328
|
def list_all_files
|
318
329
|
list = Dir.glob("#{self.local_path}/**/*", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) or (x =~/^#{self.local_path}\/\.cnvrgignore.conflict*/) and not (x =~/^#{self.local_path}\/\.cnvrgignore/) }
|
319
330
|
list_ignore = self.get_ignore_list()
|
@@ -423,15 +434,13 @@ module Cnvrg
|
|
423
434
|
CLI.is_response_success(response,false)
|
424
435
|
return response
|
425
436
|
end
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
end
|
437
|
+
|
438
|
+
def compare_idx_download(all_files: false, desired_commit: nil)
|
439
|
+
current_commit = self.last_local_commit
|
430
440
|
ignore_list = self.send_ignore_list()
|
431
|
-
|
432
|
-
CLI.is_response_success(response,false)
|
433
|
-
return response
|
441
|
+
return Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/download_status", 'POST', {current_commit: current_commit, ignore:ignore_list,all_files:all_files, desired_commit: desired_commit.presence})
|
434
442
|
end
|
443
|
+
|
435
444
|
def current_status(new_branch)
|
436
445
|
commit=last_local_commit
|
437
446
|
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status_current", 'POST', {current_commit: commit,new_branch: new_branch})
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -263,7 +263,8 @@ parameters:
|
|
263
263
|
:http_open_timeout => 60, :retry_limit => 20)
|
264
264
|
use_accelerate_endpoint = true
|
265
265
|
else
|
266
|
-
|
266
|
+
|
267
|
+
endpoint = Cnvrg::Helpers.decrypt(key,iv, files["endpoint_url"])
|
267
268
|
client = Aws::S3::Client.new(
|
268
269
|
:access_key_id =>access,
|
269
270
|
:secret_access_key => secret,
|
data/lib/cnvrg/version.rb
CHANGED