cnvrg 0.0.1520000 → 0.0.1530000
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/Images.rb +35 -6
- data/lib/cnvrg/cli.rb +88 -27
- data/lib/cnvrg/datafiles.rb +75 -25
- data/lib/cnvrg/files.rb +60 -72
- data/lib/cnvrg/helpers.rb +13 -0
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: dfd7e3915a7ca0c235ac844cbb19de4ecd985ff8
|
4
|
+
data.tar.gz: e5435e7a078993757e7e0bcc937632cd73d16ffd
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 05388102f35ca1434e624cdab9bdad58c52e28594fcc9b5a18edb9ff7a2953341fbcf9e141a1bd751fcfb9136763afba99a0f3adfe841600c84375d674dd7580
|
7
|
+
data.tar.gz: 9350e72abedd72550692d0b4fa65f9f6c95ad5f56d9f343480ce3c360a02b8ee369ba9fc9dac37dda8837f1c581893c3d2755bc37a46c3ddbf0dae23e4cad349
|
data/lib/cnvrg/Images.rb
CHANGED
@@ -2,6 +2,7 @@ require 'fileutils'
|
|
2
2
|
require 'cnvrg/files'
|
3
3
|
require 'docker'
|
4
4
|
require 'net/ssh'
|
5
|
+
require 'mimemagic'
|
5
6
|
|
6
7
|
|
7
8
|
module Cnvrg
|
@@ -95,18 +96,46 @@ module Cnvrg
|
|
95
96
|
|
96
97
|
|
97
98
|
end
|
98
|
-
def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
|
99
|
-
|
100
|
-
|
101
|
-
|
99
|
+
def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path)
|
100
|
+
response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
|
101
|
+
is_base:is_base,image_extend:image_extend,
|
102
|
+
python3:python3})
|
103
|
+
|
102
104
|
return response
|
103
105
|
end
|
106
|
+
def self.create_new_custom_image_with_docker(type,owner,image_name,is_public,is_base,image_extend,python3,tar_path,files)
|
107
|
+
file_name = File.basename tar_path
|
108
|
+
file_size = File.size(tar_path).to_f
|
109
|
+
mime_type = MimeMagic.by_path(tar_path)
|
110
|
+
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
111
|
+
|
112
|
+
upload_resp = Cnvrg::API.request("/users/#{owner}/images/upload_docker", 'POST_FILE', {absolute_path: tar_path, relative_path: tar_path,
|
113
|
+
file_name: file_name, file_size: file_size,
|
114
|
+
file_content_type: content_type,
|
115
|
+
image_name:image_name,is_public:is_public,
|
116
|
+
is_base:is_base,image_extend:image_extend,
|
117
|
+
python3:python3 })
|
118
|
+
|
119
|
+
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
120
|
+
path = upload_resp["result"]["path"]
|
121
|
+
|
122
|
+
s3_res = files.upload_small_files_s3(path, tar_path, content_type)
|
123
|
+
if s3_res
|
124
|
+
image_slug = upload_resp["result"]["id"]
|
125
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{image_slug}/build", 'POST', {instance_type:type,image_extend:image_extend})
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
|
132
|
+
|
104
133
|
def self.revoke_custom_new_image(owner,slug)
|
105
134
|
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
|
106
135
|
return response
|
107
136
|
end
|
108
|
-
def self.commit_custom_image(owner,slug)
|
109
|
-
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", '
|
137
|
+
def self.commit_custom_image(owner,slug,logs)
|
138
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'POST', {image_logs:logs})
|
110
139
|
return response
|
111
140
|
end
|
112
141
|
def self.ssh_to_machine(resp)
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -783,7 +783,6 @@ module Cnvrg
|
|
783
783
|
working_dir = "#{Dir.pwd}/#{dataset_name}"
|
784
784
|
@dataset = Dataset.new(working_dir)
|
785
785
|
@dataset.generate_idx()
|
786
|
-
say "Downloading data", Thor::Shell::Color::BLUE
|
787
786
|
|
788
787
|
|
789
788
|
download_data(false, false, path = working_dir, in_dir=false)
|
@@ -899,6 +898,8 @@ module Cnvrg
|
|
899
898
|
log_end(0)
|
900
899
|
return true
|
901
900
|
end
|
901
|
+
say "Downloading data", Thor::Shell::Color::BLUE
|
902
|
+
|
902
903
|
result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
|
903
904
|
|
904
905
|
delete = result["result"]["delete"]
|
@@ -1711,10 +1712,12 @@ module Cnvrg
|
|
1711
1712
|
update_count += 1
|
1712
1713
|
successful_updates<< relative_path
|
1713
1714
|
else
|
1715
|
+
say "Failed to upload: #{ File.basename(absolute_path) }", Thor::Shell::Color::RED
|
1716
|
+
|
1714
1717
|
@files.rollback_commit(commit_sha1)
|
1715
|
-
log_end(1, "
|
1718
|
+
log_end(1, "Failed to upload: #{ File.basename(absolute_path) }, rolling back")
|
1716
1719
|
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
1717
|
-
exit(
|
1720
|
+
exit(1)
|
1718
1721
|
end
|
1719
1722
|
end
|
1720
1723
|
end
|
@@ -1807,7 +1810,7 @@ module Cnvrg
|
|
1807
1810
|
@files.rollback_commit(commit_sha1)
|
1808
1811
|
end
|
1809
1812
|
rescue => e
|
1810
|
-
log_end(-1)
|
1813
|
+
log_end(-1,e.message)
|
1811
1814
|
|
1812
1815
|
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
1813
1816
|
@files.rollback_commit(commit_sha1)
|
@@ -2978,7 +2981,6 @@ module Cnvrg
|
|
2978
2981
|
def remote_notebook()
|
2979
2982
|
verify_logged_in(true)
|
2980
2983
|
log_start(__method__, args, options)
|
2981
|
-
verify_software_installed("docker")
|
2982
2984
|
|
2983
2985
|
working_dir = is_cnvrg_dir()
|
2984
2986
|
notebook_dir = options["notebook_dir"]
|
@@ -3077,7 +3079,6 @@ module Cnvrg
|
|
3077
3079
|
begin
|
3078
3080
|
verify_logged_in(true)
|
3079
3081
|
log_start(__method__, args, options)
|
3080
|
-
verify_software_installed("docker")
|
3081
3082
|
project_dir = is_cnvrg_dir()
|
3082
3083
|
|
3083
3084
|
image = is_project_with_docker(project_dir)
|
@@ -3141,7 +3142,6 @@ module Cnvrg
|
|
3141
3142
|
begin
|
3142
3143
|
verify_logged_in(true)
|
3143
3144
|
log_start(__method__, args, options)
|
3144
|
-
verify_software_installed("docker")
|
3145
3145
|
system = options["system"] || false
|
3146
3146
|
|
3147
3147
|
|
@@ -3206,7 +3206,6 @@ module Cnvrg
|
|
3206
3206
|
begin
|
3207
3207
|
verify_logged_in(true)
|
3208
3208
|
log_start(__method__, args, options)
|
3209
|
-
verify_software_installed("docker")
|
3210
3209
|
|
3211
3210
|
project_dir = is_cnvrg_dir()
|
3212
3211
|
|
@@ -3377,7 +3376,6 @@ module Cnvrg
|
|
3377
3376
|
begin
|
3378
3377
|
verify_logged_in(true)
|
3379
3378
|
log_start(__method__, args, options)
|
3380
|
-
verify_software_installed("docker")
|
3381
3379
|
remote = options["remote"] || false
|
3382
3380
|
project_dir = is_cnvrg_dir()
|
3383
3381
|
|
@@ -3449,7 +3447,6 @@ module Cnvrg
|
|
3449
3447
|
begin
|
3450
3448
|
verify_logged_in(false)
|
3451
3449
|
log_start(__method__, args, options)
|
3452
|
-
verify_software_installed("docker")
|
3453
3450
|
image = is_project_with_docker(Dir.pwd)
|
3454
3451
|
if image and image.is_docker
|
3455
3452
|
container = image.get_container
|
@@ -3501,7 +3498,6 @@ module Cnvrg
|
|
3501
3498
|
begin
|
3502
3499
|
verify_logged_in(false)
|
3503
3500
|
log_start(__method__, args, options)
|
3504
|
-
verify_software_installed("docker")
|
3505
3501
|
image = is_project_with_docker(Dir.pwd)
|
3506
3502
|
if image and image.is_docker
|
3507
3503
|
container = image.get_container
|
@@ -3566,10 +3562,12 @@ module Cnvrg
|
|
3566
3562
|
method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
|
3567
3563
|
method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
|
3568
3564
|
method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
|
3565
|
+
method_option :docker_path, :type => :string, :aliases => ["--docker_path"], :default => ""
|
3566
|
+
|
3569
3567
|
|
3570
3568
|
desc 'create_custom_image', 'run commands inside containers', :hide=>true
|
3571
3569
|
|
3572
|
-
def
|
3570
|
+
def build_image(image_name)
|
3573
3571
|
begin
|
3574
3572
|
verify_logged_in(false)
|
3575
3573
|
log_start(__method__, args, options)
|
@@ -3580,12 +3578,30 @@ module Cnvrg
|
|
3580
3578
|
public = options["public"]
|
3581
3579
|
base = options["base"]
|
3582
3580
|
python3 = options["python3"]
|
3581
|
+
docker_path = options["docker_path"]
|
3583
3582
|
owner = CLI.get_owner
|
3584
3583
|
checks = Helpers.checkmark()
|
3584
|
+
tar_path = nil
|
3585
|
+
if !docker_path.nil? and !docker_path.empty?
|
3586
|
+
docker_path = File.absolute_path(docker_path)
|
3587
|
+
#create tar of the docker path: it could be a docker file, and it could be a docker folder
|
3588
|
+
tar_path = File.expand_path('~')+"/.cnvrg/tmp/docker_#{File.basename docker_path}.tar.gz"
|
3589
|
+
resp = create_docker_tar(docker_path,tar_path)
|
3590
|
+
if !resp
|
3591
|
+
say "Couldn't create tar from docker path", Thor::Shell::Color::RED
|
3592
|
+
FileUtils.rm_rf tar_path
|
3593
|
+
exit(1)
|
3594
|
+
end
|
3595
|
+
files = Cnvrg::Files.new(owner, "")
|
3596
|
+
resp = Images.create_new_custom_image_with_docker(instance_type,owner,image_name,public,base,image_extend,python3,tar_path,files)
|
3597
|
+
if resp
|
3598
|
+
end
|
3599
|
+
else
|
3600
|
+
say "Creating machine for your custom image, this may take a few moments...", Thor::Shell::Color::BLUE
|
3601
|
+
resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3,nil)
|
3585
3602
|
|
3586
|
-
|
3603
|
+
end
|
3587
3604
|
|
3588
|
-
resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
|
3589
3605
|
if Cnvrg::CLI.is_response_success(resp,false)
|
3590
3606
|
image_slug = resp["result"]["slug"]
|
3591
3607
|
container = resp["result"]["machine_c"]
|
@@ -3597,21 +3613,52 @@ module Cnvrg
|
|
3597
3613
|
Images.revoke_custom_new_image(owner, image_slug)
|
3598
3614
|
log_end(-1,"Couldn't connect to machine,aborting")
|
3599
3615
|
end
|
3600
|
-
say "run command until ctrl + c
|
3616
|
+
say "run command until ctrl + c or quit is initiated", Thor::Shell::Color::BLUE
|
3601
3617
|
begin
|
3618
|
+
logs = []
|
3602
3619
|
|
3603
3620
|
while true
|
3604
3621
|
command = ask("$>")
|
3605
|
-
|
3622
|
+
logs << { time: Time.now,
|
3623
|
+
message: command,
|
3624
|
+
type: "stdout"
|
3625
|
+
}
|
3626
|
+
if command.eql? "quit"
|
3627
|
+
say "Commiting Image..", Thor::Shell::Color::BLUE
|
3628
|
+
break
|
3629
|
+
end
|
3630
|
+
res = ssh.exec_command(command)
|
3631
|
+
begin
|
3632
|
+
res_parsed = JSON.parse(res)
|
3633
|
+
res = res_parsed.join(",")
|
3634
|
+
end
|
3635
|
+
|
3636
|
+
puts res
|
3637
|
+
logs << { time: Time.now,
|
3638
|
+
message: res,
|
3639
|
+
type: "stdout"
|
3640
|
+
}
|
3641
|
+
logs.flatten!
|
3642
|
+
|
3606
3643
|
end
|
3607
3644
|
|
3608
3645
|
rescue SignalException
|
3609
3646
|
say "Commiting Image..", Thor::Shell::Color::BLUE
|
3610
3647
|
|
3611
3648
|
end
|
3612
|
-
resp = Images.commit_custom_image(owner,image_slug)
|
3649
|
+
resp = Images.commit_custom_image(owner,image_slug,logs)
|
3613
3650
|
if Cnvrg::CLI.is_response_success(resp,false)
|
3614
3651
|
say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
|
3652
|
+
else
|
3653
|
+
if image_slug
|
3654
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3655
|
+
end
|
3656
|
+
if ssh
|
3657
|
+
ssh.close_ssh()
|
3658
|
+
end
|
3659
|
+
say "Image couldn't be commited, rolling back changes", Thor::Shell::Color::RED
|
3660
|
+
|
3661
|
+
exit(1)
|
3615
3662
|
end
|
3616
3663
|
if ssh
|
3617
3664
|
ssh.close_ssh()
|
@@ -3655,7 +3702,6 @@ module Cnvrg
|
|
3655
3702
|
begin
|
3656
3703
|
verify_logged_in(false)
|
3657
3704
|
log_start(__method__, args, options)
|
3658
|
-
verify_software_installed("docker")
|
3659
3705
|
working_dir = is_cnvrg_dir
|
3660
3706
|
install_file = options["install"] || nil
|
3661
3707
|
if !install_file.nil?
|
@@ -3730,7 +3776,6 @@ module Cnvrg
|
|
3730
3776
|
def commit_image
|
3731
3777
|
verify_logged_in(true)
|
3732
3778
|
log_start(__method__, args, options)
|
3733
|
-
verify_software_installed("docker")
|
3734
3779
|
|
3735
3780
|
begin
|
3736
3781
|
image = is_project_with_docker(Dir.pwd)
|
@@ -3776,7 +3821,6 @@ module Cnvrg
|
|
3776
3821
|
def sync_image(docker=false)
|
3777
3822
|
verify_logged_in(true)
|
3778
3823
|
log_start(__method__, args, options)
|
3779
|
-
verify_software_installed("docker")
|
3780
3824
|
is_public = options["is_public"] || false
|
3781
3825
|
is_base = options["is_base"] || false
|
3782
3826
|
message = options["message"] || ""
|
@@ -3848,7 +3892,6 @@ module Cnvrg
|
|
3848
3892
|
def upload_image(image_id, is_public, is_base, *message)
|
3849
3893
|
verify_logged_in(true)
|
3850
3894
|
log_start(__method__, args, options)
|
3851
|
-
verify_software_installed("docker")
|
3852
3895
|
image = Docker::Image.get(image_id)
|
3853
3896
|
project_home = get_project_home
|
3854
3897
|
@project = Project.new(project_home)
|
@@ -4197,7 +4240,6 @@ module Cnvrg
|
|
4197
4240
|
def upload_cnvrg_image(image_name)
|
4198
4241
|
verify_logged_in(false)
|
4199
4242
|
log_start(__method__, args, options)
|
4200
|
-
verify_software_installed("docker")
|
4201
4243
|
owner = Cnvrg::CLI.get_owner()
|
4202
4244
|
|
4203
4245
|
path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
|
@@ -4648,8 +4690,11 @@ module Cnvrg
|
|
4648
4690
|
if response.nil?
|
4649
4691
|
if !Cnvrg::Helpers.internet_connection?
|
4650
4692
|
say("<%= color('Error:You seems to be offline', RED) %>")
|
4693
|
+
end
|
4694
|
+
if should_exit
|
4695
|
+
exit(1)
|
4651
4696
|
else
|
4652
|
-
|
4697
|
+
return false
|
4653
4698
|
end
|
4654
4699
|
elsif response["status"]!= 200
|
4655
4700
|
error = response['message']
|
@@ -4746,10 +4791,11 @@ module Cnvrg
|
|
4746
4791
|
logfile_old = File.expand_path('~') +"/.cnvrg/log_#{date}.log"
|
4747
4792
|
count+=1
|
4748
4793
|
end
|
4749
|
-
|
4750
|
-
|
4751
|
-
|
4752
|
-
|
4794
|
+
if File.exist? logfile_old
|
4795
|
+
@files = Cnvrg::Files.new(Cnvrg::CLI.get_owner, "")
|
4796
|
+
@files.upload_log_file(logfile_old, "log_#{date}.log", yesterday)
|
4797
|
+
FileUtils.remove logfile_old
|
4798
|
+
end
|
4753
4799
|
|
4754
4800
|
end
|
4755
4801
|
$LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
|
@@ -5063,6 +5109,21 @@ module Cnvrg
|
|
5063
5109
|
return relative
|
5064
5110
|
end
|
5065
5111
|
end
|
5112
|
+
def create_docker_tar(docker_path,tar_path)
|
5113
|
+
begin
|
5114
|
+
|
5115
|
+
if File.directory? (docker_path)
|
5116
|
+
`cd #{docker_path} && tar -czf #{tar_path} . `
|
5117
|
+
else
|
5118
|
+
dir_name = File.dirname docker_path
|
5119
|
+
`cd #{dir_name} && tar -czf #{tar_path} #{File.basename(docker_path)}`
|
5120
|
+
end
|
5121
|
+
rescue => e
|
5122
|
+
puts "Exception while compressing docker path: #{e.message}"
|
5123
|
+
end
|
5124
|
+
|
5125
|
+
return $?.success?
|
5126
|
+
end
|
5066
5127
|
|
5067
5128
|
def create_tar(path_in, path_out, tar_files,no_compression=false)
|
5068
5129
|
#The cd is meant for cases when running cnvrg data uplaod not in the main folder
|
data/lib/cnvrg/datafiles.rb
CHANGED
@@ -64,13 +64,14 @@ module Cnvrg
|
|
64
64
|
|
65
65
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_tar_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
66
66
|
commit_sha1: commit_sha1, file_name: file_name,
|
67
|
-
file_size: file_size, file_content_type: content_type, sha1: sha1
|
67
|
+
file_size: file_size, file_content_type: content_type, sha1: sha1,
|
68
|
+
new_version:true})
|
68
69
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
69
70
|
path = upload_resp["result"]["path"]
|
70
71
|
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
71
72
|
if s3_res
|
72
|
-
Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
|
73
|
-
|
73
|
+
# Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
|
74
|
+
# blob_id: upload_resp["result"]["id"]})
|
74
75
|
return true
|
75
76
|
end
|
76
77
|
else
|
@@ -283,16 +284,29 @@ module Cnvrg
|
|
283
284
|
home_dir = File.expand_path('~')
|
284
285
|
log_file = "#{home_dir}/.cnvrg/tmp/upload_#{File.basename(file_path)}.log"
|
285
286
|
|
286
|
-
|
287
|
-
|
288
|
-
|
289
|
-
|
287
|
+
|
288
|
+
split = body.split("\n")
|
289
|
+
key = split[0]
|
290
|
+
iv = split[1]
|
291
|
+
|
292
|
+
access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["new_a"])
|
293
|
+
|
294
|
+
secret = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["new_s"])
|
295
|
+
|
296
|
+
region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
|
297
|
+
|
298
|
+
bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
|
299
|
+
|
300
|
+
|
301
|
+
client = Aws::S3::Client.new(:access_key_id => access,
|
302
|
+
:secret_access_key =>secret,
|
303
|
+
:region => region,
|
290
304
|
:logger => Logger.new(log_file),
|
291
305
|
:http_open_timeout => 60, :retry_limit => 20,
|
292
306
|
:http_wire_trace => true)
|
293
307
|
s3 = Aws::S3::Resource.new(client: client)
|
294
308
|
|
295
|
-
resp = s3.bucket(
|
309
|
+
resp = s3.bucket(bucket).
|
296
310
|
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
297
311
|
upload_file(file_path, {:use_accelerate_endpoint => true})
|
298
312
|
# s4cmd_path = upload_resp["result"]["path_s4cmd"]
|
@@ -407,15 +421,33 @@ module Cnvrg
|
|
407
421
|
response = http.request request
|
408
422
|
body = response.read_body
|
409
423
|
end
|
410
|
-
|
411
|
-
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
|
424
|
+
split = body.split("\n")
|
425
|
+
key = split[0]
|
426
|
+
iv = split[1]
|
427
|
+
|
428
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
429
|
+
|
430
|
+
secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
|
431
|
+
|
432
|
+
session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
|
433
|
+
region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
|
434
|
+
|
435
|
+
bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
|
436
|
+
key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
|
437
|
+
|
438
|
+
|
439
|
+
|
440
|
+
client = Aws::S3::Client.new(
|
441
|
+
:access_key_id =>access,
|
442
|
+
:secret_access_key => secret,
|
443
|
+
:session_token => session,
|
444
|
+
:region => region,
|
445
|
+
:http_open_timeout => 60, :retry_limit => 20
|
446
|
+
)
|
447
|
+
|
416
448
|
File.open(project_home+"/"+absolute_path, 'wb') do |file|
|
417
|
-
resp =
|
418
|
-
key:
|
449
|
+
resp = client.get_object({bucket:bucket,
|
450
|
+
key:key}, target: file)
|
419
451
|
end
|
420
452
|
return true
|
421
453
|
end
|
@@ -428,7 +460,7 @@ module Cnvrg
|
|
428
460
|
|
429
461
|
def download_data_file(commit_sha1, dataset_home)
|
430
462
|
begin
|
431
|
-
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1})
|
463
|
+
res = Cnvrg::API.request(@base_resource + "download_data_file", 'POST', {commit_sha1: commit_sha1,new_version:true})
|
432
464
|
Cnvrg::CLI.is_response_success(res, false)
|
433
465
|
if res["result"]
|
434
466
|
download_resp = res
|
@@ -445,16 +477,34 @@ module Cnvrg
|
|
445
477
|
response = http.request request
|
446
478
|
body = response.read_body
|
447
479
|
end
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
480
|
+
split = body.split("\n")
|
481
|
+
key = split[0]
|
482
|
+
iv = split[1]
|
483
|
+
|
484
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
485
|
+
|
486
|
+
secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
|
487
|
+
|
488
|
+
session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
|
489
|
+
region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
|
490
|
+
|
491
|
+
bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
|
492
|
+
key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
|
493
|
+
|
494
|
+
|
495
|
+
|
496
|
+
client = Aws::S3::Client.new(
|
497
|
+
:access_key_id =>access,
|
498
|
+
:secret_access_key => secret,
|
499
|
+
:session_token => session,
|
500
|
+
:region => region,
|
501
|
+
:http_open_timeout => 60, :retry_limit => 20
|
502
|
+
)
|
503
|
+
|
454
504
|
|
455
505
|
File.open(dataset_home+"/"+filename, 'wb') do |file|
|
456
|
-
resp =
|
457
|
-
key:
|
506
|
+
resp = client.get_object({bucket: bucket,
|
507
|
+
key: key }, target: file)
|
458
508
|
end
|
459
509
|
return filename
|
460
510
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -26,7 +26,8 @@ module Cnvrg
|
|
26
26
|
|
27
27
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
28
28
|
commit_sha1: commit_sha1, file_name: file_name,
|
29
|
-
file_size: file_size, file_content_type: content_type, sha1: sha1
|
29
|
+
file_size: file_size, file_content_type: content_type, sha1: sha1,
|
30
|
+
new_version:true})
|
30
31
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
31
32
|
path = upload_resp["result"]["path"]
|
32
33
|
if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
|
@@ -241,7 +242,6 @@ module Cnvrg
|
|
241
242
|
|
242
243
|
|
243
244
|
sts_path = upload_resp["result"]["path_sts"]
|
244
|
-
s4cmd_path = upload_resp["result"]["path_s4cmd"]
|
245
245
|
|
246
246
|
uri = URI.parse(sts_path)
|
247
247
|
http_object = Net::HTTP.new(uri.host, uri.port)
|
@@ -253,77 +253,45 @@ module Cnvrg
|
|
253
253
|
response = http.request request
|
254
254
|
body = response.read_body
|
255
255
|
end
|
256
|
+
split = body.split("\n")
|
257
|
+
key = split[0]
|
258
|
+
iv = split[1]
|
256
259
|
|
257
|
-
|
258
|
-
is_python = false
|
259
|
-
s4cmd_suc = false
|
260
|
-
s4cmd_install_suc = false
|
261
|
-
# python_version=`python --version > /dev/null 2>&1`; is_python=$?.success?
|
262
|
-
# if is_python
|
263
|
-
#
|
264
|
-
# s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
|
265
|
-
# if !s4cmd_suc
|
266
|
-
# `pip install s4cmd > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
|
267
|
-
# end
|
268
|
-
#
|
269
|
-
# end
|
270
|
-
if !s4cmd_suc and !s4cmd_install_suc
|
271
|
-
s3 = Aws::S3::Resource.new(
|
272
|
-
:access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
|
273
|
-
:secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
|
274
|
-
:session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
|
275
|
-
:region => URLcrypt.decrypt(upload_resp["result"]["region"]))
|
276
|
-
resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
|
277
|
-
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
278
|
-
upload_file(file_path, {:use_accelerate_endpoint => true})
|
279
|
-
else
|
280
|
-
s4cmd_uri = URI.parse(s4cmd_path)
|
281
|
-
s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
|
282
|
-
s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
|
283
|
-
s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
|
284
|
-
|
285
|
-
s4cmd_body = ""
|
286
|
-
s4cmd_http_object.start do |http|
|
287
|
-
response = http.request s4cmd_request
|
288
|
-
s4cmd_body = response.read_body
|
289
|
-
end
|
290
|
-
s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
|
291
|
-
aws_access_key_id=aws_access_key_id,
|
292
|
-
aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
|
293
|
-
aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
|
294
|
-
aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
|
295
|
-
aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
|
296
|
-
|
297
|
-
|
298
|
-
tmp = Tempfile.new('s4cmd.py')
|
299
|
-
tmp << s4cmd_new_body
|
300
|
-
tmp.flush
|
301
|
-
tmp.close
|
302
|
-
#
|
303
|
-
is_success = false
|
304
|
-
count = 0
|
305
|
-
while !is_success and count <3
|
306
|
-
resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
|
307
|
-
is_success =$?.success?
|
308
|
-
count +=1
|
260
|
+
access = Cnvrg::Helpers.decrypt(key, iv, upload_resp["result"]["sts_a"])
|
309
261
|
|
310
|
-
|
311
|
-
|
262
|
+
secret = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_s"])
|
263
|
+
|
264
|
+
session = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["sts_st"])
|
265
|
+
region = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["region"])
|
266
|
+
|
267
|
+
bucket = Cnvrg::Helpers.decrypt(key,iv, upload_resp["result"]["bucket"])
|
312
268
|
|
313
|
-
|
269
|
+
|
270
|
+
|
271
|
+
client = Aws::S3::Client.new(
|
272
|
+
:access_key_id =>access,
|
273
|
+
:secret_access_key => secret,
|
274
|
+
:session_token => session,
|
275
|
+
:region => region,
|
276
|
+
:http_open_timeout => 60, :retry_limit => 20
|
277
|
+
)
|
278
|
+
s3 = Aws::S3::Resource.new(client: client)
|
279
|
+
resp = s3.bucket(bucket).
|
280
|
+
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
281
|
+
upload_file(file_path, {:use_accelerate_endpoint => true})
|
314
282
|
|
315
283
|
return resp
|
316
284
|
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
return false
|
285
|
+
rescue => e
|
286
|
+
puts e
|
287
|
+
puts e.backtrace
|
288
|
+
return false
|
322
289
|
|
323
290
|
end
|
324
291
|
return true
|
325
292
|
|
326
|
-
|
293
|
+
end
|
294
|
+
|
327
295
|
|
328
296
|
def upload_small_files_s3(url_path, file_path, content_type)
|
329
297
|
url = URI.parse(url_path)
|
@@ -370,7 +338,8 @@ module Cnvrg
|
|
370
338
|
|
371
339
|
def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
|
372
340
|
begin
|
373
|
-
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
|
341
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path,
|
342
|
+
commit_sha1: commit_sha1,new_version:true})
|
374
343
|
Cnvrg::CLI.is_response_success(res, false)
|
375
344
|
if res["result"]
|
376
345
|
download_resp = res
|
@@ -388,20 +357,39 @@ module Cnvrg
|
|
388
357
|
response = http.request request
|
389
358
|
body = response.read_body
|
390
359
|
end
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
|
395
|
-
|
396
|
-
|
360
|
+
split = body.split("\n")
|
361
|
+
key = split[0]
|
362
|
+
iv = split[1]
|
363
|
+
|
364
|
+
access = Cnvrg::Helpers.decrypt(key, iv, download_resp["result"]["sts_a"])
|
365
|
+
|
366
|
+
secret = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_s"])
|
367
|
+
|
368
|
+
session = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["sts_st"])
|
369
|
+
region = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["region"])
|
370
|
+
|
371
|
+
bucket = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["bucket"])
|
372
|
+
key = Cnvrg::Helpers.decrypt(key,iv, download_resp["result"]["key"])
|
373
|
+
|
374
|
+
|
375
|
+
|
376
|
+
client = Aws::S3::Client.new(
|
377
|
+
:access_key_id =>access,
|
378
|
+
:secret_access_key => secret,
|
379
|
+
:session_token => session,
|
380
|
+
:region => region,
|
381
|
+
:http_open_timeout => 60, :retry_limit => 20
|
382
|
+
)
|
383
|
+
|
397
384
|
File.open(project_home+"/"+absolute_path, 'wb') do |file|
|
398
|
-
resp =
|
399
|
-
key:
|
385
|
+
resp = client.get_object({bucket:bucket,
|
386
|
+
key:key}, target: file)
|
400
387
|
end
|
401
388
|
return true
|
402
389
|
end
|
403
390
|
|
404
391
|
rescue => e
|
392
|
+
puts e
|
405
393
|
return false
|
406
394
|
|
407
395
|
end
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -171,6 +171,19 @@ parameters:
|
|
171
171
|
Process.clock_gettime(Process::CLOCK_MONOTONIC, :microsecond)
|
172
172
|
end
|
173
173
|
|
174
|
+
def decrypt(key,iv,str)
|
175
|
+
cipher = OpenSSL::Cipher.new("aes-256-cbc").decrypt
|
176
|
+
cipher.key = key
|
177
|
+
cipher.iv = Base64.decode64 iv.encode('ascii-8bit')
|
178
|
+
|
179
|
+
result = Base64.decode64 (str.encode('ascii-8bit'))
|
180
|
+
result = cipher.update(result)
|
181
|
+
result << cipher.final
|
182
|
+
|
183
|
+
return result
|
184
|
+
|
185
|
+
end
|
186
|
+
|
174
187
|
# memory
|
175
188
|
#
|
176
189
|
def get_mem(pid)
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.1530000
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-07-
|
12
|
+
date: 2017-07-30 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|