cnvrg 0.0.15 → 0.0.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +5 -9
- data/lib/cnvrg/Images.rb +15 -76
- data/lib/cnvrg/api.rb +4 -7
- data/lib/cnvrg/cli.rb +527 -2149
- data/lib/cnvrg/experiment.rb +10 -19
- data/lib/cnvrg/files.rb +208 -302
- data/lib/cnvrg/helpers.rb +1 -42
- data/lib/cnvrg/job.rb +1 -0
- data/lib/cnvrg/project.rb +15 -55
- data/lib/cnvrg/version.rb +1 -2
- data/lib/cnvrg.rb +1 -0
- metadata +17 -48
- data/lib/cnvrg/data.rb +0 -72
- data/lib/cnvrg/datafiles.rb +0 -509
- data/lib/cnvrg/dataset.rb +0 -296
- data/lib/cnvrg/ssh.rb +0 -95
data/lib/cnvrg/files.rb
CHANGED
@@ -1,14 +1,11 @@
|
|
1
1
|
require 'mimemagic'
|
2
2
|
require 'aws-sdk'
|
3
3
|
require 'URLcrypt'
|
4
|
-
require 'tempfile'
|
5
4
|
|
6
5
|
module Cnvrg
|
7
6
|
class Files
|
8
7
|
|
9
|
-
LARGE_FILE=1024*1024*
|
10
|
-
MULTIPART_SPLIT=10000000
|
11
|
-
|
8
|
+
LARGE_FILE=1024*1024*100
|
12
9
|
attr_reader :base_resource
|
13
10
|
|
14
11
|
def initialize(owner, project_slug)
|
@@ -22,11 +19,9 @@ module Cnvrg
|
|
22
19
|
file_size = File.size(absolute_path).to_f
|
23
20
|
mime_type = MimeMagic.by_path(absolute_path)
|
24
21
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
25
|
-
sha1 = Digest::SHA1.file(absolute_path).hexdigest
|
26
|
-
|
27
22
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
28
23
|
commit_sha1: commit_sha1, file_name: file_name,
|
29
|
-
file_size: file_size, file_content_type: content_type
|
24
|
+
file_size: file_size, file_content_type: content_type})
|
30
25
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
31
26
|
path = upload_resp["result"]["path"]
|
32
27
|
if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
|
@@ -34,27 +29,24 @@ module Cnvrg
|
|
34
29
|
else
|
35
30
|
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
36
31
|
end
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
#
|
43
|
-
# return is_suc
|
32
|
+
if s3_res
|
33
|
+
Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
|
34
|
+
blob_id: upload_resp["result"]["id"]})
|
35
|
+
return true
|
36
|
+
end
|
44
37
|
end
|
45
38
|
return false
|
46
39
|
end
|
47
|
-
|
48
|
-
def upload_log_file(absolute_path, relative_path, log_date)
|
40
|
+
def upload_log_file(absolute_path, relative_path,log_date)
|
49
41
|
file_name = File.basename relative_path
|
50
42
|
file_size = File.size(absolute_path).to_f
|
51
43
|
content_type = "text/x-log"
|
52
44
|
upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
53
|
-
|
54
|
-
|
45
|
+
file_name: file_name,log_date:log_date,
|
46
|
+
file_size: file_size, file_content_type: content_type})
|
55
47
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
56
|
-
|
57
|
-
|
48
|
+
path = upload_resp["result"]["path"]
|
49
|
+
s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
|
58
50
|
end
|
59
51
|
if s3_res
|
60
52
|
return true
|
@@ -62,19 +54,18 @@ module Cnvrg
|
|
62
54
|
return false
|
63
55
|
|
64
56
|
end
|
65
|
-
|
66
|
-
def upload_exec_file(absolute_path, image_name, commit_id)
|
57
|
+
def upload_exec_file(absolute_path,image_name,commit_id)
|
67
58
|
file_name = File.basename absolute_path
|
68
59
|
file_size = File.size(absolute_path).to_f
|
69
60
|
content_type = "application/zip"
|
70
61
|
begin
|
71
62
|
upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
|
72
63
|
file_name: file_name,
|
73
|
-
image_name:
|
64
|
+
image_name:image_name,
|
74
65
|
file_size: file_size,
|
75
66
|
file_content_type: content_type,
|
76
67
|
project_slug: @project_slug,
|
77
|
-
commit_id:
|
68
|
+
commit_id:commit_id})
|
78
69
|
# puts upload_resp
|
79
70
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
80
71
|
if upload_resp["result"]["image"] == -1
|
@@ -84,20 +75,21 @@ module Cnvrg
|
|
84
75
|
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
85
76
|
|
86
77
|
end
|
87
|
-
|
88
|
-
|
89
|
-
end
|
90
|
-
return false
|
91
|
-
rescue SignalException
|
92
|
-
|
93
|
-
say "\nAborting"
|
94
|
-
exit(1)
|
78
|
+
if s3_res
|
79
|
+
return upload_resp["result"]["id"]
|
95
80
|
end
|
81
|
+
return false
|
82
|
+
rescue SignalException
|
96
83
|
|
84
|
+
say "\nAborting"
|
85
|
+
exit(1)
|
97
86
|
end
|
98
87
|
|
88
|
+
end
|
99
89
|
|
100
|
-
|
90
|
+
|
91
|
+
|
92
|
+
def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
|
101
93
|
file_name = File.basename absolute_path
|
102
94
|
file_size = File.size(absolute_path).to_f
|
103
95
|
if is_base
|
@@ -107,42 +99,40 @@ module Cnvrg
|
|
107
99
|
content_type = "application/gzip"
|
108
100
|
end
|
109
101
|
begin
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
if
|
130
|
-
commit_resp
|
131
|
-
|
132
|
-
|
133
|
-
else
|
134
|
-
return false
|
135
|
-
end
|
136
|
-
|
102
|
+
upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload", 'POST_FILE', {relative_path: absolute_path,
|
103
|
+
file_name: file_name,
|
104
|
+
image_name: image_name,
|
105
|
+
file_size: file_size,
|
106
|
+
file_content_type: content_type,
|
107
|
+
is_public: is_public,
|
108
|
+
project_slug: @project_slug,
|
109
|
+
commit_id:commit_id ,
|
110
|
+
dpkg: dpkg,
|
111
|
+
libraries: libraries,
|
112
|
+
bash_history: bash,
|
113
|
+
commit_message:message,
|
114
|
+
is_base: is_base})
|
115
|
+
# puts upload_resp
|
116
|
+
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
117
|
+
path = upload_resp["result"]["path"]
|
118
|
+
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
119
|
+
if s3_res
|
120
|
+
commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
|
121
|
+
if Cnvrg::CLI.is_response_success(commit_resp, false)
|
122
|
+
return commit_resp["result"]["image"]
|
123
|
+
else
|
124
|
+
return false
|
137
125
|
end
|
126
|
+
|
138
127
|
end
|
139
|
-
|
140
|
-
|
128
|
+
end
|
129
|
+
return false
|
130
|
+
rescue =>e
|
131
|
+
puts e
|
141
132
|
end
|
142
133
|
|
143
134
|
end
|
144
|
-
|
145
|
-
def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
|
135
|
+
def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
|
146
136
|
file_name = File.basename absolute_path
|
147
137
|
file_size = File.size(absolute_path).to_f
|
148
138
|
if is_base
|
@@ -153,16 +143,16 @@ module Cnvrg
|
|
153
143
|
end
|
154
144
|
begin
|
155
145
|
upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
146
|
+
file_name: file_name,
|
147
|
+
image_name: image_name,
|
148
|
+
file_size: file_size,
|
149
|
+
file_content_type: content_type,
|
150
|
+
is_public: is_public,
|
151
|
+
dpkg: dpkg,
|
152
|
+
libraries: libraries,
|
153
|
+
bash_history: bash,
|
154
|
+
commit_message:message,
|
155
|
+
is_base: is_base})
|
166
156
|
# puts upload_resp
|
167
157
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
168
158
|
path = upload_resp["result"]["path"]
|
@@ -178,7 +168,8 @@ module Cnvrg
|
|
178
168
|
end
|
179
169
|
end
|
180
170
|
return false
|
181
|
-
rescue =>
|
171
|
+
rescue =>e
|
172
|
+
puts e
|
182
173
|
end
|
183
174
|
|
184
175
|
end
|
@@ -197,6 +188,7 @@ module Cnvrg
|
|
197
188
|
|
198
189
|
return true
|
199
190
|
rescue => e
|
191
|
+
puts e
|
200
192
|
return false
|
201
193
|
end
|
202
194
|
|
@@ -210,260 +202,174 @@ module Cnvrg
|
|
210
202
|
|
211
203
|
def upload_large_files_s3(upload_resp, file_path)
|
212
204
|
begin
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
# speed_tmp = Tempfile.new('speed')
|
224
|
-
# speed_tmp << speed_body
|
225
|
-
# speed_tmp.flush
|
226
|
-
# speed_tmp.close
|
227
|
-
# count = 0
|
228
|
-
# sum = 0
|
229
|
-
# speed_res = `python #{speed_tmp.path} --json`
|
230
|
-
# if !speed_res.nil? and !speed_res.empty?
|
231
|
-
# upload = JSON.parse(speed_res)["upload"]
|
232
|
-
# if !upload.nil?
|
233
|
-
# up_spped = (upload/1000000).round(2)
|
234
|
-
# end
|
235
|
-
#
|
236
|
-
#
|
237
|
-
# file_size = File.size(file_path)/1048576
|
238
|
-
# est_up = ((file_size*8)/up_spped)/60
|
239
|
-
# puts est_up
|
240
|
-
# end
|
241
|
-
|
242
|
-
|
243
|
-
sts_path = upload_resp["result"]["path_sts"]
|
244
|
-
s4cmd_path = upload_resp["result"]["path_s4cmd"]
|
245
|
-
|
246
|
-
uri = URI.parse(sts_path)
|
247
|
-
http_object = Net::HTTP.new(uri.host, uri.port)
|
248
|
-
http_object.use_ssl = true if uri.scheme == 'https'
|
249
|
-
request = Net::HTTP::Get.new(sts_path)
|
250
|
-
|
251
|
-
body = ""
|
252
|
-
http_object.start do |http|
|
253
|
-
response = http.request request
|
254
|
-
body = response.read_body
|
255
|
-
end
|
256
|
-
|
257
|
-
URLcrypt::key = [body].pack('H*')
|
258
|
-
is_python = false
|
259
|
-
s4cmd_suc = false
|
260
|
-
s4cmd_install_suc = false
|
261
|
-
# python_version=`python --version > /dev/null 2>&1`; is_python=$?.success?
|
262
|
-
# if is_python
|
263
|
-
#
|
264
|
-
# s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
|
265
|
-
# if !s4cmd_suc
|
266
|
-
# `pip install s4cmd > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
|
267
|
-
# end
|
268
|
-
#
|
269
|
-
# end
|
270
|
-
if !s4cmd_suc and !s4cmd_install_suc
|
271
|
-
s3 = Aws::S3::Resource.new(
|
272
|
-
:access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
|
273
|
-
:secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
|
274
|
-
:session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
|
275
|
-
:region => URLcrypt.decrypt(upload_resp["result"]["region"]))
|
276
|
-
resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
|
277
|
-
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
278
|
-
upload_file(file_path, {:use_accelerate_endpoint => true})
|
279
|
-
else
|
280
|
-
s4cmd_uri = URI.parse(s4cmd_path)
|
281
|
-
s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
|
282
|
-
s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
|
283
|
-
s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
|
284
|
-
|
285
|
-
s4cmd_body = ""
|
286
|
-
s4cmd_http_object.start do |http|
|
287
|
-
response = http.request s4cmd_request
|
288
|
-
s4cmd_body = response.read_body
|
289
|
-
end
|
290
|
-
s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
|
291
|
-
aws_access_key_id=aws_access_key_id,
|
292
|
-
aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
|
293
|
-
aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
|
294
|
-
aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
|
295
|
-
aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
|
296
|
-
|
297
|
-
|
298
|
-
tmp = Tempfile.new('s4cmd.py')
|
299
|
-
tmp << s4cmd_new_body
|
300
|
-
tmp.flush
|
301
|
-
tmp.close
|
302
|
-
#
|
303
|
-
is_success = false
|
304
|
-
count = 0
|
305
|
-
while !is_success and count <3
|
306
|
-
resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
|
307
|
-
is_success =$?.success?
|
308
|
-
count +=1
|
309
|
-
|
310
|
-
end
|
311
|
-
resp= is_success
|
312
|
-
|
313
|
-
end
|
314
|
-
|
315
|
-
return resp
|
316
|
-
|
317
|
-
rescue => e
|
318
|
-
if File.exist? tmp
|
319
|
-
FileUtils.rm_rf [tmp]
|
320
|
-
end
|
321
|
-
return false
|
322
|
-
|
205
|
+
sts_path = upload_resp["result"]["path_sts"]
|
206
|
+
uri = URI.parse(sts_path)
|
207
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
208
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
209
|
+
request = Net::HTTP::Get.new(sts_path)
|
210
|
+
|
211
|
+
body = ""
|
212
|
+
http_object.start do |http|
|
213
|
+
response = http.request request
|
214
|
+
body = response.read_body
|
323
215
|
end
|
324
|
-
|
216
|
+
URLcrypt::key = [body].pack('H*')
|
217
|
+
s3 = Aws::S3::Resource.new(
|
218
|
+
:access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
|
219
|
+
:secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
|
220
|
+
:session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
|
221
|
+
:region => URLcrypt.decrypt(upload_resp["result"]["region"]))
|
222
|
+
resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
|
223
|
+
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
224
|
+
upload_file(file_path)
|
225
|
+
return resp
|
226
|
+
rescue =>e
|
227
|
+
puts e
|
228
|
+
return false
|
325
229
|
|
326
230
|
end
|
231
|
+
return true
|
327
232
|
|
328
|
-
|
329
|
-
url = URI.parse(url_path)
|
330
|
-
file = File.open(file_path, "rb")
|
331
|
-
body = file.read
|
332
|
-
begin
|
333
|
-
Net::HTTP.start(url.host) do |http|
|
334
|
-
http.send_request("PUT", url.request_uri, body, {
|
335
|
-
"content-type" => content_type,
|
336
|
-
})
|
337
|
-
end
|
338
|
-
return true
|
339
|
-
rescue Interrupt
|
340
|
-
return false
|
341
|
-
rescue
|
342
|
-
return false
|
343
|
-
end
|
344
|
-
end
|
233
|
+
end
|
345
234
|
|
346
|
-
|
347
|
-
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
235
|
+
def upload_small_files_s3(url_path, file_path, content_type)
|
236
|
+
url = URI.parse(url_path)
|
237
|
+
file = File.open(file_path, "rb")
|
238
|
+
body = file.read
|
239
|
+
begin
|
240
|
+
Net::HTTP.start(url.host) do |http|
|
241
|
+
http.send_request("PUT", url.request_uri, body, {
|
242
|
+
"content-type" => content_type,
|
243
|
+
})
|
352
244
|
end
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
return Cnvrg::CLI.is_response_success(response, false)
|
359
|
-
end
|
360
|
-
|
361
|
-
def delete_dir(absolute_path, relative_path, commit_sha1)
|
362
|
-
response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
363
|
-
return Cnvrg::CLI.is_response_success(response, false)
|
245
|
+
return true
|
246
|
+
rescue Interrupt
|
247
|
+
return false
|
248
|
+
rescue
|
249
|
+
return false
|
364
250
|
end
|
251
|
+
end
|
365
252
|
|
366
|
-
|
367
|
-
|
368
|
-
|
253
|
+
def upload_url(file_path)
|
254
|
+
response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
|
255
|
+
if Cnvrg::CLI.is_response_success(response, false)
|
256
|
+
return response
|
257
|
+
else
|
258
|
+
return nil
|
369
259
|
end
|
370
260
|
|
371
|
-
|
372
|
-
begin
|
373
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
374
|
-
Cnvrg::CLI.is_response_success(res, false)
|
375
|
-
if res["result"]
|
376
|
-
download_resp = res
|
377
|
-
filename = download_resp["result"]["filename"]
|
378
|
-
|
379
|
-
absolute_path += ".conflict" if conflict
|
380
|
-
sts_path = download_resp["result"]["path_sts"]
|
381
|
-
uri = URI.parse(sts_path)
|
382
|
-
http_object = Net::HTTP.new(uri.host, uri.port)
|
383
|
-
http_object.use_ssl = true if uri.scheme == 'https'
|
384
|
-
request = Net::HTTP::Get.new(sts_path)
|
385
|
-
|
386
|
-
body = ""
|
387
|
-
http_object.start do |http|
|
388
|
-
response = http.request request
|
389
|
-
body = response.read_body
|
390
|
-
end
|
391
|
-
URLcrypt::key = [body].pack('H*')
|
392
|
-
s3 = Aws::S3::Client.new(
|
393
|
-
:access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
|
394
|
-
:secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
|
395
|
-
:session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
|
396
|
-
:region => URLcrypt.decrypt(download_resp["result"]["region"]))
|
397
|
-
File.open(project_home+"/"+absolute_path, 'wb') do |file|
|
398
|
-
resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
|
399
|
-
key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
|
400
|
-
end
|
401
|
-
return true
|
402
|
-
end
|
261
|
+
end
|
403
262
|
|
404
|
-
|
405
|
-
|
263
|
+
def delete_file(absolute_path, relative_path, commit_sha1)
|
264
|
+
response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
265
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
266
|
+
end
|
406
267
|
|
407
|
-
|
408
|
-
|
268
|
+
def delete_dir(absolute_path, relative_path, commit_sha1)
|
269
|
+
response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
270
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
271
|
+
end
|
409
272
|
|
410
|
-
|
273
|
+
def create_dir(absolute_path, relative_path, commit_sha1)
|
274
|
+
response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
275
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
276
|
+
end
|
277
|
+
def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
|
278
|
+
begin
|
411
279
|
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
412
280
|
Cnvrg::CLI.is_response_success(res, false)
|
413
281
|
if res["result"]
|
414
|
-
|
415
|
-
|
416
|
-
|
417
|
-
|
282
|
+
download_resp = res
|
283
|
+
filename = download_resp["result"]["filename"]
|
284
|
+
|
285
|
+
absolute_path += ".conflict" if conflict
|
286
|
+
sts_path = download_resp["result"]["path_sts"]
|
287
|
+
uri = URI.parse(sts_path)
|
288
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
289
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
290
|
+
request = Net::HTTP::Get.new(sts_path)
|
291
|
+
|
292
|
+
body = ""
|
293
|
+
http_object.start do |http|
|
294
|
+
response = http.request request
|
295
|
+
body = response.read_body
|
296
|
+
end
|
297
|
+
URLcrypt::key = [body].pack('H*')
|
298
|
+
s3 = Aws::S3::Client.new(
|
299
|
+
:access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
|
300
|
+
:secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
|
301
|
+
:session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
|
302
|
+
:region => URLcrypt.decrypt(download_resp["result"]["region"]))
|
303
|
+
File.open(project_home+"/"+absolute_path, 'wb') do |file|
|
304
|
+
resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
|
305
|
+
key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
|
306
|
+
end
|
307
|
+
return true
|
308
|
+
end
|
418
309
|
|
419
|
-
|
420
|
-
|
310
|
+
rescue =>e
|
311
|
+
puts e
|
312
|
+
puts e.backtrace
|
313
|
+
return false
|
421
314
|
|
422
|
-
File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
|
423
|
-
file.write open(res["link"]).read
|
424
|
-
end
|
425
|
-
else
|
426
|
-
return false
|
427
|
-
end
|
428
|
-
return true
|
429
315
|
end
|
430
|
-
|
431
|
-
|
432
|
-
|
316
|
+
end
|
317
|
+
def download_file(absolute_path, relative_path, project_home, conflict=false)
|
318
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
319
|
+
Cnvrg::CLI.is_response_success(res, false)
|
320
|
+
if res["result"]
|
321
|
+
res = res["result"]
|
322
|
+
return false if res["link"].empty? or res["filename"].empty?
|
323
|
+
filename = res["filename"]
|
324
|
+
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
325
|
+
|
326
|
+
FileUtils.mkdir_p project_home + "/" + file_location
|
327
|
+
filename += ".conflict" if conflict
|
328
|
+
|
329
|
+
File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
|
330
|
+
file.write open(res["link"]).read
|
331
|
+
end
|
332
|
+
else
|
333
|
+
return false
|
433
334
|
end
|
335
|
+
return true
|
336
|
+
end
|
434
337
|
|
435
|
-
|
436
|
-
|
437
|
-
|
338
|
+
def download_dir(absolute_path, relative_path, project_home)
|
339
|
+
FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
|
340
|
+
end
|
341
|
+
def revoke_download_dir(absolute_path, relative_path, project_home)
|
342
|
+
puts FileUtils.rmtree("#{absolute_path}")
|
343
|
+
end
|
438
344
|
|
439
|
-
|
440
|
-
|
441
|
-
|
345
|
+
def revoke_download_file(project_home,absolute_path,filename,conflict=false)
|
346
|
+
begin
|
347
|
+
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
442
348
|
|
443
|
-
|
444
|
-
|
445
|
-
|
446
|
-
|
447
|
-
|
448
|
-
end
|
349
|
+
filename += ".conflict" if conflict
|
350
|
+
FileUtils.remove("#{file_location}/#{filename}")
|
351
|
+
return true
|
352
|
+
rescue
|
353
|
+
return false
|
449
354
|
end
|
355
|
+
end
|
450
356
|
|
451
|
-
|
357
|
+
def start_commit(new_branch)
|
452
358
|
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
|
359
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug,new_branch:new_branch,
|
360
|
+
username: @owner})
|
361
|
+
Cnvrg::CLI.is_response_success(response)
|
362
|
+
return response
|
363
|
+
end
|
458
364
|
|
459
|
-
|
460
|
-
|
461
|
-
|
462
|
-
|
365
|
+
def end_commit(commit_sha1)
|
366
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
|
367
|
+
return response
|
368
|
+
end
|
463
369
|
|
464
|
-
|
465
|
-
|
466
|
-
|
467
|
-
end
|
370
|
+
def rollback_commit(commit_sha1)
|
371
|
+
response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
|
372
|
+
Cnvrg::CLI.is_response_success(response, false)
|
468
373
|
end
|
469
374
|
end
|
375
|
+
end
|