cnvrg 0.6.2 → 0.6.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +20 -10
- data/lib/cnvrg/datafiles.rb +5 -2
- data/lib/cnvrg/dataset.rb +10 -2
- data/lib/cnvrg/experiment.rb +0 -8
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1eb2a526f32419697a93b9927b01ec1327588db
|
4
|
+
data.tar.gz: cbc3517d2341d6a1905190a6a6f99bfa1e1df20c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9287cf89d3440a871737dd037c2bc5b3dc6ebc2457ea31cfbd1156d022891460977d44a5993f06620d396ec9604ce5786ba63e957672773dac32a259a2792b45
|
7
|
+
data.tar.gz: 66a9c63f4adccb2720d9cbee9b7f17ac79cedda0e9389bd9300b830c02292ad03259eec1c42e427ea095df1bffe1fa5496d1239d96fe37ce5b207041def87b82
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -1938,10 +1938,13 @@ module Cnvrg
|
|
1938
1938
|
def upload_data_new(new_branch, verbose,sync,force, tags, chunk_size)
|
1939
1939
|
commit = invoke :start_commit_data,[], :new_branch=> new_branch, :direct=>false, :force =>force, :chunk_size => chunk_size
|
1940
1940
|
upload_res = invoke :upload_data_files,[commit],:new_branch=>new_branch, :verbose =>verbose, :force =>force, :sync =>sync, :chunk_size => chunk_size
|
1941
|
-
if
|
1941
|
+
if upload_res < 0
|
1942
1942
|
return
|
1943
1943
|
end
|
1944
|
-
|
1944
|
+
# if !upload_res
|
1945
|
+
# return
|
1946
|
+
# end
|
1947
|
+
invoke :end_commit_data,[commit, success: true, uploaded_files: upload_res] , :new_branch=>new_branch, :force =>force
|
1945
1948
|
if tags
|
1946
1949
|
log_message('Uploading Tags', Thor::Shell::Color::BLUE)
|
1947
1950
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
@@ -2028,7 +2031,7 @@ module Cnvrg
|
|
2028
2031
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
2029
2032
|
method_option :force, :type => :boolean, :aliases => ["-f","--force"], :default => false
|
2030
2033
|
|
2031
|
-
def end_commit_data(commit, success: true)
|
2034
|
+
def end_commit_data(commit, success: true, uploaded_files: 0)
|
2032
2035
|
begin
|
2033
2036
|
verify_logged_in(true)
|
2034
2037
|
log_start(__method__, args, options)
|
@@ -2036,13 +2039,19 @@ module Cnvrg
|
|
2036
2039
|
@dataset = Dataset.new(dataset_dir)
|
2037
2040
|
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2038
2041
|
force = options["force"] || false
|
2039
|
-
|
2040
|
-
resp = @files.end_commit(commit, force, success: success)
|
2042
|
+
resp = @files.end_commit(commit, force, success: success, uploaded_files: uploaded_files)
|
2041
2043
|
if (resp and resp["result"])
|
2042
2044
|
check = Helpers.checkmark
|
2043
|
-
|
2044
|
-
|
2045
|
-
|
2045
|
+
if resp["result"]["new_commit"].blank?
|
2046
|
+
@dataset.revert_next_commit
|
2047
|
+
log_message("#{check} Dataset is up to date", Thor::Shell::Color::GREEN)
|
2048
|
+
else
|
2049
|
+
log_message("#{check} Data files were updated successfully", Thor::Shell::Color::GREEN)
|
2050
|
+
@dataset.set_next_commit(resp["result"]["commit_sha1"]) # no need fo rboth methods FIX
|
2051
|
+
@dataset.remove_next_commit()
|
2052
|
+
|
2053
|
+
end
|
2054
|
+
|
2046
2055
|
end
|
2047
2056
|
rescue => e
|
2048
2057
|
puts e
|
@@ -2133,10 +2142,11 @@ module Cnvrg
|
|
2133
2142
|
if upload_resp.is_a? Cnvrg::Result
|
2134
2143
|
log_message(upload_resp.msg, upload_resp.msg_color)
|
2135
2144
|
|
2136
|
-
return
|
2145
|
+
return -1
|
2137
2146
|
|
2138
2147
|
else
|
2139
2148
|
files_uploaded += upload_resp
|
2149
|
+
next if upload_resp == 0
|
2140
2150
|
end
|
2141
2151
|
|
2142
2152
|
temp_tree.each do |k,v|
|
@@ -2144,7 +2154,7 @@ module Cnvrg
|
|
2144
2154
|
end
|
2145
2155
|
@dataset.write_idx(new_tree, new_commit)
|
2146
2156
|
end
|
2147
|
-
return files_uploaded
|
2157
|
+
return files_uploaded
|
2148
2158
|
rescue Exception => e
|
2149
2159
|
puts e
|
2150
2160
|
return false
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -29,6 +29,9 @@ module Cnvrg
|
|
29
29
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch})
|
30
30
|
return Cnvrg::Result.new(false, "Failed to upload files") unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
31
31
|
results = upload_resp['result'].with_indifferent_access
|
32
|
+
if results['files'].blank?
|
33
|
+
return 0
|
34
|
+
end
|
32
35
|
props = Cnvrg::Helpers.get_s3_props(results)
|
33
36
|
client = props[:client]
|
34
37
|
bucket = props[:bucket]
|
@@ -756,9 +759,9 @@ module Cnvrg
|
|
756
759
|
|
757
760
|
end
|
758
761
|
|
759
|
-
def end_commit(commit_sha1,force, success: true)
|
762
|
+
def end_commit(commit_sha1,force, success: true, uploaded_files: 0 )
|
760
763
|
begin
|
761
|
-
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success})
|
764
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1,force:force, success: success, uploaded_files: uploaded_files})
|
762
765
|
Cnvrg::CLI.is_response_success(response, true)
|
763
766
|
return response
|
764
767
|
rescue => e
|
data/lib/cnvrg/dataset.rb
CHANGED
@@ -501,8 +501,16 @@ module Cnvrg
|
|
501
501
|
idx[:commit] = idx_hash[:next_commit]
|
502
502
|
idx[:tree] = idx_hash[:tree]
|
503
503
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
504
|
-
|
505
|
-
|
504
|
+
end
|
505
|
+
def revert_next_commit()
|
506
|
+
if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
|
507
|
+
return nil
|
508
|
+
end
|
509
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
510
|
+
idx = Hash.new()
|
511
|
+
idx[:commit] = idx_hash[:commit]
|
512
|
+
idx[:tree] = idx_hash[:tree]
|
513
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
506
514
|
end
|
507
515
|
def get_current_commit()
|
508
516
|
if !File.exist? "#{self.local_path}/.cnvrg/idx.yml"
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -139,14 +139,6 @@ module Cnvrg
|
|
139
139
|
def restart_spot_instance
|
140
140
|
|
141
141
|
restart = false
|
142
|
-
#TODO: remove this later
|
143
|
-
fall_number = rand(1..20)
|
144
|
-
if [5,15].include? fall_number
|
145
|
-
return true
|
146
|
-
else
|
147
|
-
return false
|
148
|
-
end
|
149
|
-
|
150
142
|
begin
|
151
143
|
url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
|
152
144
|
req = Net::HTTP::Get.new(url.to_s)
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-08-
|
12
|
+
date: 2018-08-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|