cnvrg 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,72 @@
1
+ require 'cnvrg/cli'
2
+ require 'thor'
3
+ class SubCommandBase < Thor
4
+ def self.banner(command, namespace = nil, subcommand = false)
5
+ "#{basename} #{command.usage}"
6
+ end
7
+
8
+ def self.subcommand_prefix
9
+ self.name.gsub(%r{.*::}, '').gsub(%r{^[A-Z]}) { |match| match[0].downcase }.gsub(%r{[A-Z]}) { |match| "-#{match[0].downcase}" }
10
+ end
11
+ end
12
+
13
+ module Cnvrg
14
+
15
+ class Data < SubCommandBase
16
+ class_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
17
+ desc "data init", "init data folder"
18
+ method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
19
+ def init
20
+ cli = Cnvrg::CLI.new()
21
+ public = options["public"]
22
+ cli.init_data(public)
23
+ end
24
+ desc "data upload", "upload data folder"
25
+ method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
26
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
27
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
28
+ method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
29
+
30
+ def upload
31
+ cli = Cnvrg::CLI.new()
32
+ ignore = options["ignore"]
33
+ verbose = options["verbose"]
34
+ sync = options["sync"]
35
+ no_compression = options["no_compression"]
36
+ cli.upload_data_tar(ignore, verbose,sync,no_compression)
37
+ end
38
+ desc 'data download', 'pull data'
39
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
40
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
41
+
42
+ def download()
43
+ cli = Cnvrg::CLI.new()
44
+ verbose = options["verbose"]
45
+ sync = options["sync"]
46
+
47
+ cli.download_data(verbose,sync,Dir.pwd)
48
+
49
+ end
50
+ desc 'data clone', 'clone datset'
51
+ def clone(dataset_url)
52
+ cli = Cnvrg::CLI.new()
53
+ cli.clone_data(dataset_url)
54
+
55
+ end
56
+ desc 'data list', 'list of datasets'
57
+ def list()
58
+ cli = Cnvrg::CLI.new()
59
+
60
+ cli.list_dataset()
61
+
62
+ end
63
+ desc 'data commits', 'pull data'
64
+
65
+ def commits()
66
+ cli = Cnvrg::CLI.new()
67
+ cli.list_dataset_commits()
68
+
69
+ end
70
+
71
+ end
72
+ end
@@ -0,0 +1,509 @@
1
+ require 'mimemagic'
2
+ require 'aws-sdk'
3
+ require 'URLcrypt'
4
+
5
+ module Cnvrg
6
+ class Datafiles
7
+
8
+ LARGE_FILE=1024*1024*5
9
+ MULTIPART_SPLIT=10000000
10
+
11
+ attr_reader :base_resource
12
+
13
+ def initialize(owner, dataset_slug)
14
+ @dataset_slug = dataset_slug
15
+ @owner = owner
16
+ @base_resource = "users/#{owner}/datasets/#{dataset_slug}/"
17
+ end
18
+
19
+ def upload_file(absolute_path, relative_path, commit_sha1)
20
+ file_name = File.basename relative_path
21
+ file_size = File.size(absolute_path).to_f
22
+ mime_type = MimeMagic.by_path(absolute_path)
23
+ content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
24
+ sha1 = Digest::SHA1.file(absolute_path).hexdigest
25
+
26
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
27
+ commit_sha1: commit_sha1, file_name: file_name,
28
+ file_size: file_size, file_content_type: content_type,sha1:sha1})
29
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
30
+ path = upload_resp["result"]["path"]
31
+ if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
32
+ s3_res = upload_large_files_s3(upload_resp, absolute_path)
33
+ else
34
+ s3_res = upload_small_files_s3(path, absolute_path, content_type)
35
+ end
36
+ if s3_res
37
+ Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
38
+ blob_id: upload_resp["result"]["id"]})
39
+ return true
40
+ end
41
+ end
42
+ return false
43
+ end
44
+ def upload_tar_file(absolute_path, relative_path, commit_sha1)
45
+ begin
46
+ file_name = File.basename relative_path
47
+ file_size = File.size(absolute_path).to_f
48
+ mime_type = MimeMagic.by_path(absolute_path)
49
+ content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
50
+ sha1 = Digest::SHA1.file(absolute_path).hexdigest
51
+
52
+ upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
53
+ commit_sha1: commit_sha1, file_name: file_name,
54
+ file_size: file_size, file_content_type: content_type,sha1:sha1})
55
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
56
+ path = upload_resp["result"]["path"]
57
+ s3_res = upload_large_files_s3(upload_resp, absolute_path)
58
+ if s3_res
59
+ Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
60
+ blob_id: upload_resp["result"]["id"]})
61
+ return true
62
+ end
63
+ else
64
+ return false
65
+ end
66
+ rescue => e
67
+ puts e.message
68
+ return false
69
+ end
70
+
71
+ end
72
+ def upload_log_file(absolute_path, relative_path,log_date)
73
+ file_name = File.basename relative_path
74
+ file_size = File.size(absolute_path).to_f
75
+ content_type = "text/x-log"
76
+ upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
77
+ file_name: file_name,log_date:log_date,
78
+ file_size: file_size, file_content_type: content_type})
79
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
80
+ path = upload_resp["result"]["path"]
81
+ s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
82
+ end
83
+ if s3_res
84
+ return true
85
+ end
86
+ return false
87
+
88
+ end
89
+ def upload_exec_file(absolute_path,image_name,commit_id)
90
+ file_name = File.basename absolute_path
91
+ file_size = File.size(absolute_path).to_f
92
+ content_type = "application/zip"
93
+ begin
94
+ upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
95
+ file_name: file_name,
96
+ image_name:image_name,
97
+ file_size: file_size,
98
+ file_content_type: content_type,
99
+ project_slug: @project_slug,
100
+ commit_id:commit_id})
101
+ # puts upload_resp
102
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
103
+ if upload_resp["result"]["image"] == -1
104
+ return -1
105
+ end
106
+ path = upload_resp["result"]["path"]
107
+ s3_res = upload_small_files_s3(path, absolute_path, content_type)
108
+
109
+ end
110
+ if s3_res
111
+ return upload_resp["result"]["id"]
112
+ end
113
+ return false
114
+ rescue SignalException
115
+
116
+ say "\nAborting"
117
+ exit(1)
118
+ end
119
+
120
+ end
121
+
122
+
123
+
124
+ def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
125
+ file_name = File.basename absolute_path
126
+ file_size = File.size(absolute_path).to_f
127
+ if is_base
128
+
129
+ content_type = "application/zip"
130
+ else
131
+ content_type = "application/gzip"
132
+ end
133
+ begin
134
+ upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
135
+ file_name: file_name,
136
+ image_name: image_name,
137
+ file_size: file_size,
138
+ file_content_type: content_type,
139
+ is_public: is_public,
140
+ project_slug: @project_slug,
141
+ commit_id:commit_id ,
142
+ dpkg: dpkg,
143
+ py2: libraries,
144
+ py3: libraries,
145
+
146
+ bash_history: bash,
147
+ commit_message:message,
148
+ is_base: is_base})
149
+ # puts upload_resp
150
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
151
+ path = upload_resp["result"]["path"]
152
+ s3_res = upload_small_files_s3(path, absolute_path, content_type)
153
+ if s3_res
154
+ commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
155
+ if Cnvrg::CLI.is_response_success(commit_resp, false)
156
+ return commit_resp["result"]["image"]
157
+ else
158
+ return false
159
+ end
160
+
161
+ end
162
+ end
163
+ return false
164
+ rescue =>e
165
+ end
166
+
167
+ end
168
+ def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
169
+ file_name = File.basename absolute_path
170
+ file_size = File.size(absolute_path).to_f
171
+ if is_base
172
+
173
+ content_type = "application/zip"
174
+ else
175
+ content_type = "application/gzip"
176
+ end
177
+ begin
178
+ upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
179
+ file_name: file_name,
180
+ image_name: image_name,
181
+ file_size: file_size,
182
+ file_content_type: content_type,
183
+ is_public: is_public,
184
+ dpkg: dpkg,
185
+ libraries: libraries,
186
+ bash_history: bash,
187
+ commit_message:message,
188
+ is_base: is_base})
189
+ # puts upload_resp
190
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
191
+ path = upload_resp["result"]["path"]
192
+ s3_res = upload_small_files_s3(path, absolute_path, content_type)
193
+ if s3_res
194
+ commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
195
+ if Cnvrg::CLI.is_response_success(commit_resp, false)
196
+ return commit_resp["result"]["image"]
197
+ else
198
+ return false
199
+ end
200
+
201
+ end
202
+ end
203
+ return false
204
+ rescue =>e
205
+ end
206
+
207
+ end
208
+
209
+ def download_image(file_path_to_store, image_slug, owner)
210
+
211
+
212
+ download_resp = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/" + "download", 'GET')
213
+ path = download_resp["result"]["path"]
214
+
215
+ if Cnvrg::CLI.is_response_success(download_resp, false)
216
+ begin
217
+ open(file_path_to_store, 'wb') do |file|
218
+ file << open(path).read
219
+ end
220
+
221
+ return true
222
+ rescue => e
223
+ return false
224
+ end
225
+
226
+ return true
227
+ else
228
+ return false
229
+ end
230
+
231
+
232
+ end
233
+
234
+ def upload_large_files_s3(upload_resp, file_path)
235
+ begin
236
+ sts_path = upload_resp["result"]["path_sts"]
237
+ # s4cmd_path = upload_resp["result"]["path_s4cmd"]
238
+
239
+ uri = URI.parse(sts_path)
240
+ http_object = Net::HTTP.new(uri.host, uri.port)
241
+ http_object.use_ssl = true if uri.scheme == 'https'
242
+ request = Net::HTTP::Get.new(sts_path)
243
+
244
+ body = ""
245
+ http_object.start do |http|
246
+ response = http.request request
247
+ body = response.read_body
248
+ end
249
+ is_python = false
250
+ s4cmd_suc = false
251
+ s4cmd_install_suc = false
252
+
253
+ URLcrypt::key = [body].pack('H*')
254
+
255
+ # python_version=`python --version > /dev/null 2>&1` ; is_python=$?.success?
256
+ # if is_python
257
+ #
258
+ # s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1` ; s4cmd_suc=$?.success?
259
+ # if !s4cmd_suc
260
+ # `pip install s4cmd > /dev/null 2>&1` ; s4cmd_install_suc=$?.success?
261
+ # end
262
+ #
263
+ # end
264
+ resp = false
265
+
266
+ if !s4cmd_suc and !s4cmd_install_suc
267
+ s3 = Aws::S3::Resource.new(
268
+ :access_key_id => URLcrypt.decrypt(upload_resp["result"]["new_a"]),
269
+ :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["new_s"]),
270
+ :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
271
+ resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
272
+ object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
273
+ upload_file(file_path,{:use_accelerate_endpoint=>true})
274
+
275
+ # else
276
+
277
+ # s4cmd_uri = URI.parse(s4cmd_path)
278
+ # s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
279
+ # s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
280
+ # s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
281
+ #
282
+ # s4cmd_body = ""
283
+ # s4cmd_http_object.start do |http|
284
+ # response = http.request s4cmd_request
285
+ # s4cmd_body = response.read_body
286
+ # end
287
+ #
288
+ # s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
289
+ # aws_access_key_id=aws_access_key_id,
290
+ # aws_secret_access_key=aws_secret_access_key)"," self.client = self.boto3.client('s3',
291
+ # aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
292
+ # aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
293
+ # aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
294
+ #
295
+ # tmp = Tempfile.new('s4cmd.py')
296
+ # tmp << s4cmd_new_body
297
+ # tmp.flush
298
+ # tmp.close
299
+ #
300
+ # is_success = false
301
+ # count = 0
302
+ # while !is_success and count <3
303
+ # resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1`
304
+ # is_success =$?.success?
305
+ # count +=1
306
+ #
307
+ # end
308
+ # resp= is_success
309
+
310
+ end
311
+
312
+ return resp
313
+
314
+ rescue => e
315
+ puts e.message
316
+ # if File.exist? tmp
317
+ # FileUtils.rm_rf [tmp]
318
+ # end
319
+ return false
320
+
321
+ end
322
+ return true
323
+
324
+ end
325
+
326
+
327
+ def upload_small_files_s3(url_path, file_path, content_type)
328
+ url = URI.parse(url_path)
329
+ file = File.open(file_path, "rb")
330
+ body = file.read
331
+ begin
332
+ Net::HTTP.start(url.host) do |http|
333
+ http.send_request("PUT", url.request_uri, body, {
334
+ "content-type" => content_type,
335
+ })
336
+ end
337
+ return true
338
+ rescue Interrupt
339
+ return false
340
+ rescue
341
+ return false
342
+ end
343
+ end
344
+
345
+ def upload_url(file_path)
346
+ response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
347
+ if Cnvrg::CLI.is_response_success(response, false)
348
+ return response
349
+ else
350
+ return nil
351
+ end
352
+
353
+ end
354
+
355
+ def delete_file(absolute_path, relative_path, commit_sha1)
356
+ response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
357
+ return Cnvrg::CLI.is_response_success(response, false)
358
+ end
359
+
360
+ def delete_dir(absolute_path, relative_path, commit_sha1)
361
+ response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
362
+ return Cnvrg::CLI.is_response_success(response, false)
363
+ end
364
+
365
+ def create_dir(absolute_path, relative_path, commit_sha1)
366
+ response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
367
+ return Cnvrg::CLI.is_response_success(response, false)
368
+ end
369
+ def download_file_s3(absolute_path, relative_path, project_home, conflict=false,commit_sha1=nil)
370
+ begin
371
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,commit_sha1:commit_sha1})
372
+ Cnvrg::CLI.is_response_success(res, false)
373
+ if res["result"]
374
+ download_resp = res
375
+ filename = download_resp["result"]["filename"]
376
+
377
+ absolute_path += ".conflict" if conflict
378
+ sts_path = download_resp["result"]["path_sts"]
379
+ uri = URI.parse(sts_path)
380
+ http_object = Net::HTTP.new(uri.host, uri.port)
381
+ http_object.use_ssl = true if uri.scheme == 'https'
382
+ request = Net::HTTP::Get.new(sts_path)
383
+
384
+ body = ""
385
+ http_object.start do |http|
386
+ response = http.request request
387
+ body = response.read_body
388
+ end
389
+ URLcrypt::key = [body].pack('H*')
390
+ s3 = Aws::S3::Client.new(
391
+ :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
392
+ :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
393
+ :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
394
+ :region => URLcrypt.decrypt(download_resp["result"]["region"]))
395
+ File.open(project_home+"/"+absolute_path, 'wb') do |file|
396
+ resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
397
+ key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
398
+ end
399
+ return true
400
+ end
401
+
402
+ rescue =>e
403
+ return false
404
+
405
+ end
406
+ end
407
+
408
+ def download_data_file(commit_sha1,dataset_home)
409
+ begin
410
+ res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1:commit_sha1})
411
+ Cnvrg::CLI.is_response_success(res, false)
412
+ if res["result"]
413
+ download_resp = res
414
+ filename = download_resp["result"]["filename"]
415
+
416
+ sts_path = download_resp["result"]["path_sts"]
417
+ uri = URI.parse(sts_path)
418
+ http_object = Net::HTTP.new(uri.host, uri.port)
419
+ http_object.use_ssl = true if uri.scheme == 'https'
420
+ request = Net::HTTP::Get.new(sts_path)
421
+
422
+ body = ""
423
+ http_object.start do |http|
424
+ response = http.request request
425
+ body = response.read_body
426
+ end
427
+ URLcrypt::key = [body].pack('H*')
428
+ s3 = Aws::S3::Client.new(
429
+ :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
430
+ :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
431
+ :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
432
+ :region => URLcrypt.decrypt(download_resp["result"]["region"]))
433
+
434
+ File.open(dataset_home+"/"+filename, 'wb') do |file|
435
+ resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
436
+ key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
437
+ end
438
+ return filename
439
+ end
440
+
441
+ rescue =>e
442
+ return false
443
+
444
+ end
445
+ end
446
+
447
+ def download_file(absolute_path, relative_path, project_home, conflict=false)
448
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
449
+ Cnvrg::CLI.is_response_success(res, false)
450
+ if res["result"]
451
+ res = res["result"]
452
+ return false if res["link"].empty? or res["filename"].empty?
453
+ filename = res["filename"]
454
+ file_location = absolute_path.gsub(/#{filename}\/?$/, "")
455
+
456
+ FileUtils.mkdir_p project_home + "/" + file_location
457
+ filename += ".conflict" if conflict
458
+
459
+ File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
460
+ file.write open(res["link"]).read
461
+ end
462
+ else
463
+ return false
464
+ end
465
+ return true
466
+ end
467
+
468
+ def download_dir(dataset_home, absolute_path)
469
+ FileUtils.mkdir_p("#{dataset_home}/#{absolute_path}")
470
+ end
471
+ def revoke_download_dir(absolute_path)
472
+ puts FileUtils.rmtree("#{absolute_path}")
473
+ end
474
+
475
+ def revoke_download_file(absolute_path,filename,conflict=false)
476
+ begin
477
+ file_location = absolute_path.gsub(/#{filename}\/?$/, "")
478
+
479
+ filename += ".conflict" if conflict
480
+ FileUtils.remove("#{file_location}/#{filename}")
481
+ return true
482
+ rescue
483
+ return false
484
+ end
485
+ end
486
+
487
+ def start_commit(new_branch)
488
+
489
+ response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {dataset_slug: @dataset_slug,new_branch:false,
490
+ username: @owner})
491
+ Cnvrg::CLI.is_response_success(response,false)
492
+ return response
493
+ end
494
+
495
+ def end_commit(commit_sha1)
496
+ response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
497
+ return response
498
+ end
499
+ def end_commit_tar(commit_sha1,cur_idx)
500
+ response = Cnvrg::API.request("#{base_resource}/commit/end_tar", 'POST', {commit_sha1: commit_sha1,idx: cur_idx})
501
+ return response
502
+ end
503
+
504
+ def rollback_commit(commit_sha1)
505
+ response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
506
+ Cnvrg::CLI.is_response_success(response, false)
507
+ end
508
+ end
509
+ end