cnvrg 0.6.2 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +20 -10
- data/lib/cnvrg/datafiles.rb +5 -2
- data/lib/cnvrg/dataset.rb +10 -2
- data/lib/cnvrg/experiment.rb +0 -8
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1eb2a526f32419697a93b9927b01ec1327588db
|
4
|
+
data.tar.gz: cbc3517d2341d6a1905190a6a6f99bfa1e1df20c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9287cf89d3440a871737dd037c2bc5b3dc6ebc2457ea31cfbd1156d022891460977d44a5993f06620d396ec9604ce5786ba63e957672773dac32a259a2792b45
|
7
|
+
data.tar.gz: 66a9c63f4adccb2720d9cbee9b7f17ac79cedda0e9389bd9300b830c02292ad03259eec1c42e427ea095df1bffe1fa5496d1239d96fe37ce5b207041def87b82
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1938,10 +1938,13 @@ module Cnvrg
|
|
1938
1938
|
def upload_data_new(new_branch, verbose,sync,force, tags, chunk_size)
|
1939
1939
|
commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
|
1940
1940
|
upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
|
1941
|
-
if
|
1941
|
+
if upload_res < 0
|
1942
1942
|
return
|
1943
1943
|
end
|
1944
|
-
|
1944
|
+
# if !upload_res
|
1945
|
+
# return
|
1946
|
+
# end
|
1947
|
+
invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res] , :new_branch=>new_branch, :force =>force
|
1945
1948
|
if tags
|
1946
1949
|
log_message('Uploading Tags', Thor::Shell::Color::BLUE)
|
1947
1950
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
@@ -2028,7 +2031,7 @@ module Cnvrg
|
|
2028
2031
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
2029
2032
|
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
2030
2033
|
|
2031
|
-
def end_commit_data(commit, success: true)
|
2034
|
+
def end_commit_data(commit, success: true, uploaded_files: 0)
|
2032
2035
|
begin
|
2033
2036
|
verify_logged_in(true)
|
2034
2037
|
log_start(__method__, args, options)
|
@@ -2036,13 +2039,19 @@ module Cnvrg
|
|
2036
2039
|
@dataset = Dataset.new(dataset_dir)
|
2037
2040
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2038
2041
|
force = options["force"] || false
|
2039
|
-
|
2040
|
-
resp = @files.end_commit(commit, force, success: success)
|
2042
|
+
resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
|
2041
2043
|
if (resp and resp["result"])
|
2042
2044
|
check = Helpers.checkmark
|
2043
|
-
|
2044
|
-
|
2045
|
-
|
2045
|
+
if resp["result"]["new_commit"].blank?
|
2046
|
+
@dataset.revert_next_commit
|
2047
|
+
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
|
2048
|
+
else
|
2049
|
+
log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
|
2050
|
+
@dataset.set_next_commit(resp["result"]["commit_sha1"]) # no need fo rboth methods FIX
|
2051
|
+
@dataset.remove_next_commit()
|
2052
|
+
|
2053
|
+
end
|
2054
|
+
|
2046
2055
|
end
|
2047
2056
|
rescue => e
|
2048
2057
|
puts e
|
@@ -2133,10 +2142,11 @@ module Cnvrg
|
|
2133
2142
|
if upload_resp.is_a? Cnvrg::Result
|
2134
2143
|
log_message(upload_resp.msg, upload_resp.msg_color)
|
2135
2144
|
|
2136
|
-
return
|
2145
|
+
return -1
|
2137
2146
|
|
2138
2147
|
else
|
2139
2148
|
files_uploaded += upload_resp
|
2149
|
+
next if upload_resp == 0
|
2140
2150
|
end
|
2141
2151
|
|
2142
2152
|
temp_tree.each do |k,v|
|
@@ -2144,7 +2154,7 @@ module Cnvrg
|
|
2144
2154
|
end
|
2145
2155
|
@dataset.write_idx(new_tree, new_commit)
|
2146
2156
|
end
|
2147
|
-
return files_uploaded
|
2157
|
+
return files_uploaded
|
2148
2158
|
rescue Exception => e
|
2149
2159
|
puts e
|
2150
2160
|
return false
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -29,6 +29,9 @@ module Cnvrg
|
|
29
29
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
|
30
30
|
return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
31
31
|
results = upload_resp['result'].with_indifferent_access
|
32
|
+
if results['files'].blank?
|
33
|
+
return 0
|
34
|
+
end
|
32
35
|
props = Cnvrg::Helpers.get_s3_props(results)
|
33
36
|
client = props[:client]
|
34
37
|
bucket = props[:bucket]
|
@@ -756,9 +759,9 @@ module Cnvrg
|
|
756
759
|
|
757
760
|
end
|
758
761
|
|
759
|
-
def end_commit(commit_sha1,force, success: true)
|
762
|
+
def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
|
760
763
|
begin
|
761
|
-
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success})
|
764
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
|
762
765
|
Cnvrg::CLI.is_response_success(response, true)
|
763
766
|
return response
|
764
767
|
rescue => e
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -501,8 +501,16 @@ module Cnvrg
|
|
501
501
|
idx[:commit] = idx_hash[:next_commit]
|
502
502
|
idx[:tree] = idx_hash[:tree]
|
503
503
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
504
|
-
|
505
|
-
|
504
|
+
end
|
505
|
+
def revert_next_commit()
|
506
|
+
if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
|
507
|
+
return nil
|
508
|
+
end
|
509
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
510
|
+
idx = Hash.new()
|
511
|
+
idx[:commit] = idx_hash[:commit]
|
512
|
+
idx[:tree] = idx_hash[:tree]
|
513
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
506
514
|
end
|
507
515
|
def get_current_commit()
|
508
516
|
if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -139,14 +139,6 @@ module Cnvrg
|
|
139
139
|
def restart_spot_instance
|
140
140
|
|
141
141
|
restart = false
|
142
|
-
#TODO: remove this later
|
143
|
-
fall_number = rand(1..20)
|
144
|
-
if [5,15].include? fall_number
|
145
|
-
return true
|
146
|
-
else
|
147
|
-
return false
|
148
|
-
end
|
149
|
-
|
150
142
|
begin
|
151
143
|
url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
|
152
144
|
req = Net::HTTP::Get.new(url.to_s)
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-08-
|
12
|
+
date: 2018-08-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|