cnvrg 1.6.14 → 1.6.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +1 -1
- data/lib/cnvrg/cli.rb +3 -2
- data/lib/cnvrg/datafiles.rb +24 -11
- data/lib/cnvrg/downloader/clients/azure_client.rb +20 -1
- data/lib/cnvrg/downloader/clients/gcp_client.rb +6 -2
- data/lib/cnvrg/downloader/clients/s3_client.rb +2 -0
- data/lib/cnvrg/files.rb +2 -0
- data/lib/cnvrg/project.rb +3 -0
- data/lib/cnvrg/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: da2bf544fbb1248b902085a73ea2bb9ab008afd8b635b7be82dd3f6c6678ec72
|
4
|
+
data.tar.gz: 29829d584b729b86bc0637647063924927182c6a99d08f3757649516cd6dbefb
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 12901887e4d8e2a39107ec863f0b520c70dde335fbbab3626fa5fdc8f14830a3fd00b26445fea305ddc2fa943bd4f66d38eb234f10c6f107c414be6723c70f58
|
7
|
+
data.tar.gz: 9ea2f33d969fd555401a87556f765532e1fcaa9e385da8fa309a9a269b7dce2d506409013574e0f0b198c71a1d9e329063f85d32afac1f8daa69ec1aa4930143
|
data/cnvrg.gemspec
CHANGED
@@ -31,7 +31,7 @@ Gem::Specification.new do |spec|
|
|
31
31
|
spec.add_runtime_dependency 'open4', '~> 1.3', '>= 1.3.4'
|
32
32
|
spec.add_runtime_dependency 'highline', '~> 1.7', '>= 1.7.8'
|
33
33
|
spec.add_runtime_dependency 'thor', '~> 0.19.0','>=0.19.1'
|
34
|
-
spec.add_runtime_dependency 'aws-sdk', '~> 2'
|
34
|
+
spec.add_runtime_dependency 'aws-sdk', '~> 2.11.417'
|
35
35
|
spec.add_runtime_dependency 'signet', '~> 0.11.0'
|
36
36
|
spec.add_runtime_dependency 'google-cloud-env', '~> 1.2.1'
|
37
37
|
spec.add_runtime_dependency 'google-cloud-core', '~> 1.3.2'
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -845,7 +845,6 @@ module Cnvrg
|
|
845
845
|
begin
|
846
846
|
verify_logged_in(false)
|
847
847
|
log_start(__method__, args, options)
|
848
|
-
return if check_pod_restart[1] ## It means that all datasets downloaded successfully
|
849
848
|
commit = options["commit"] || commit
|
850
849
|
only_tree = options["only_tree"] || only_tree
|
851
850
|
read = options["read"] || read || false
|
@@ -1781,7 +1780,6 @@ module Cnvrg
|
|
1781
1780
|
begin
|
1782
1781
|
verify_logged_in(false)
|
1783
1782
|
log_start(__method__, args, options)
|
1784
|
-
return if check_pod_restart[0] ## It means that project downloaded successfully
|
1785
1783
|
url_parts = project_url.split("/")
|
1786
1784
|
project_index = Cnvrg::Helpers.look_for_in_path(project_url, "projects")
|
1787
1785
|
slug = url_parts[project_index + 1]
|
@@ -2155,6 +2153,7 @@ module Cnvrg
|
|
2155
2153
|
new_branch: new_branch,
|
2156
2154
|
progressbar: progressbar,
|
2157
2155
|
partial_commit: partial_commit)
|
2156
|
+
|
2158
2157
|
files_uploaded += upload_resp
|
2159
2158
|
temp_tree.each do |k, v|
|
2160
2159
|
new_tree[k] = (v.present?) ? {sha1: v.try(:fetch, :sha1, nil), commit_time: nil} : nil
|
@@ -2293,7 +2292,9 @@ module Cnvrg
|
|
2293
2292
|
:total => (to_upload.size + deleted.size),
|
2294
2293
|
:autofinish => true)
|
2295
2294
|
@files.upload_multiple_files(to_upload, commit_sha1, progress: progressbar)
|
2295
|
+
|
2296
2296
|
@files.delete_files_from_server(deleted, commit_sha1)
|
2297
|
+
|
2297
2298
|
progressbar.finish
|
2298
2299
|
res = @files.end_commit(commit_sha1, force: force, message: commit_msg)
|
2299
2300
|
unless Cnvrg::CLI.is_response_success(res, false)
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -116,6 +116,7 @@ module Cnvrg
|
|
116
116
|
|
117
117
|
def upload_multiple_files(commit_sha1, tree, threads: ParallelThreads, force: false, new_branch: false, progressbar: nil, prefix: '', partial_commit: nil)
|
118
118
|
begin
|
119
|
+
cli=CLI.new
|
119
120
|
Cnvrg::Logger.log_info("Sending Upload Files request")
|
120
121
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST_JSON', {commit_sha1: commit_sha1, tree: tree, force: force, is_branch: new_branch, partial_commit: partial_commit})
|
121
122
|
unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
@@ -132,16 +133,24 @@ module Cnvrg
|
|
132
133
|
progressbar.progress += tree.keys.length - files.length if progressbar.present?
|
133
134
|
Parallel.map((files.keys), {in_threads: threads}) do |k|
|
134
135
|
o = tree[k].merge(files[k])
|
135
|
-
upload_single_file(o)
|
136
|
+
success = upload_single_file(o)
|
137
|
+
if not success
|
138
|
+
files.except!(k)
|
139
|
+
tree.except!(k)
|
140
|
+
cli.log_message("Error while uploading file: #{o[:absolute_path]}", Thor::Shell::Color::RED)
|
141
|
+
Cnvrg::Logger.log_error_message("Error while upload single file #{o["path"]}")
|
142
|
+
end
|
136
143
|
progressbar.progress += 1 if progressbar.present?
|
137
144
|
end
|
138
145
|
blob_ids = files.values.map {|f| f['bv_id']}
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
Cnvrg::
|
144
|
-
|
146
|
+
if blob_ids.present?
|
147
|
+
dirs = tree.keys.select {|k| tree[k].nil?} || []
|
148
|
+
Cnvrg::Logger.info("Sending Upload files save")
|
149
|
+
upload_resp = Cnvrg::API.request(@base_resource + "upload_files_save", "POST", {commit: commit_sha1, blob_ids: blob_ids, dirs: dirs})
|
150
|
+
unless Cnvrg::CLI.is_response_success(upload_resp, false)
|
151
|
+
Cnvrg::Logger.log_method(bind: binding)
|
152
|
+
raise Exception.new("Got an error message from server, #{upload_resp.try(:fetch, "message")}")
|
153
|
+
end
|
145
154
|
end
|
146
155
|
Cnvrg::Logger.log_info("Upload Success")
|
147
156
|
return files.keys.length
|
@@ -155,13 +164,17 @@ module Cnvrg
|
|
155
164
|
|
156
165
|
def upload_single_file(file)
|
157
166
|
begin
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
167
|
+
cli = CLI.new
|
168
|
+
file = file.as_json
|
169
|
+
Cnvrg::Logger.log_info("Uploading #{file["absolute_path"]}")
|
170
|
+
success = @downloader.upload(file["path"], file["absolute_path"])
|
171
|
+
return false if not success
|
172
|
+
Cnvrg::Logger.log_info("#{file["absolute_path"]} uploaded.")
|
162
173
|
rescue => e
|
174
|
+
cli.log_message(e.message, Thor::Shell::Color::RED)
|
163
175
|
Cnvrg::Logger.log_error_message("Error while upload single file #{file["path"]}")
|
164
176
|
Cnvrg::Logger.log_error(e)
|
177
|
+
return false
|
165
178
|
end
|
166
179
|
end
|
167
180
|
|
@@ -20,7 +20,26 @@ module Cnvrg
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def upload(storage_path, local_path)
|
23
|
-
|
23
|
+
begin
|
24
|
+
try_upload_file(storage_path, local_path)
|
25
|
+
rescue => e
|
26
|
+
raise e
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def try_upload_file(storage_path, local_path)
|
31
|
+
c = 20
|
32
|
+
error = nil
|
33
|
+
while c > 0
|
34
|
+
begin
|
35
|
+
client.create_block_blob(@container, storage_path, File.open(local_path, "rb"))
|
36
|
+
break
|
37
|
+
rescue => e
|
38
|
+
c -= 1
|
39
|
+
error = e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
raise error if error.present?
|
24
43
|
end
|
25
44
|
|
26
45
|
def fetch_files(prefix: nil, marker: nil, limit: 10000)
|
@@ -11,7 +11,7 @@ module Cnvrg
|
|
11
11
|
@tempfile = nil
|
12
12
|
@bucket_name = Cnvrg::Helpers.decrypt(@key, @iv, bucket_name)
|
13
13
|
init_gcp_credentials
|
14
|
-
@storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials)
|
14
|
+
@storage = Google::Cloud::Storage.new(project_id: @project_id, credentials: @credentials, retries: 20, timeout: 10)
|
15
15
|
@bucket = @storage.bucket(@bucket_name)
|
16
16
|
@bucket.name
|
17
17
|
rescue => e
|
@@ -38,7 +38,11 @@ module Cnvrg
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def upload(storage_path, local_path)
|
41
|
-
|
41
|
+
begin
|
42
|
+
@bucket.create_file(local_path, storage_path)
|
43
|
+
rescue => e
|
44
|
+
raise e
|
45
|
+
end
|
42
46
|
end
|
43
47
|
end
|
44
48
|
end
|
@@ -54,6 +54,8 @@ module Cnvrg
|
|
54
54
|
# local path is fullpath for the file /home/ubuntu/user.../hazilim.py
|
55
55
|
o = aws_bucket.object(storage_path)
|
56
56
|
o.upload_file(local_path, @upload_options)
|
57
|
+
rescue => e
|
58
|
+
raise e
|
57
59
|
end
|
58
60
|
|
59
61
|
def fetch_files(prefix: nil, marker: nil, limit: 1000)
|
data/lib/cnvrg/files.rb
CHANGED
@@ -85,6 +85,7 @@ module Cnvrg
|
|
85
85
|
Cnvrg::Logger.log_info("Upload files to older server..")
|
86
86
|
return self.upload_files_old(files_list, commit_sha1, progress: progress)
|
87
87
|
end
|
88
|
+
|
88
89
|
files_list = files_list.map{|x| [x,self.parse_file(x)]}.to_h
|
89
90
|
resp = Cnvrg::API.request(@base_resource + "upload_files", 'POST', {files: files_list, commit: commit_sha1})
|
90
91
|
unless Cnvrg::CLI.is_response_success(resp, false)
|
@@ -99,6 +100,7 @@ module Cnvrg
|
|
99
100
|
begin
|
100
101
|
Cnvrg::Helpers.try_until_success{self.upload_single_file(files[file].merge(files_list[file]))}
|
101
102
|
rescue => e
|
103
|
+
|
102
104
|
Cnvrg::CLI.log_message("Failed to upload #{file}: #{e.message}", 'red')
|
103
105
|
Cnvrg::Logger.log_error(e)
|
104
106
|
Cnvrg::Logger.log_method(bind: binding)
|
data/lib/cnvrg/project.rb
CHANGED
@@ -329,10 +329,13 @@ module Cnvrg
|
|
329
329
|
def get_storage_client
|
330
330
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@slug}/client", 'GET')
|
331
331
|
if Cnvrg::CLI.is_response_success(response, false)
|
332
|
+
|
332
333
|
client_params = response['client']
|
333
334
|
else
|
335
|
+
|
334
336
|
client_params = get_storage_client_fallback
|
335
337
|
end
|
338
|
+
|
336
339
|
Cnvrg::Downloader::Client.factory(client_params)
|
337
340
|
end
|
338
341
|
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.6.
|
4
|
+
version: 1.6.15
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-01-
|
13
|
+
date: 2020-01-26 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: bundler
|
@@ -210,14 +210,14 @@ dependencies:
|
|
210
210
|
requirements:
|
211
211
|
- - "~>"
|
212
212
|
- !ruby/object:Gem::Version
|
213
|
-
version:
|
213
|
+
version: 2.11.417
|
214
214
|
type: :runtime
|
215
215
|
prerelease: false
|
216
216
|
version_requirements: !ruby/object:Gem::Requirement
|
217
217
|
requirements:
|
218
218
|
- - "~>"
|
219
219
|
- !ruby/object:Gem::Version
|
220
|
-
version:
|
220
|
+
version: 2.11.417
|
221
221
|
- !ruby/object:Gem::Dependency
|
222
222
|
name: signet
|
223
223
|
requirement: !ruby/object:Gem::Requirement
|
@@ -477,7 +477,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
477
477
|
- !ruby/object:Gem::Version
|
478
478
|
version: '0'
|
479
479
|
requirements: []
|
480
|
-
rubygems_version: 3.0.
|
480
|
+
rubygems_version: 3.0.4
|
481
481
|
signing_key:
|
482
482
|
specification_version: 4
|
483
483
|
summary: A CLI tool for interacting with cnvrg.io.
|