cnvrg 0.0.14.0 → 0.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +9 -5
- data/lib/cnvrg/Images.rb +76 -15
- data/lib/cnvrg/api.rb +7 -4
- data/lib/cnvrg/cli.rb +2148 -526
- data/lib/cnvrg/data.rb +72 -0
- data/lib/cnvrg/datafiles.rb +509 -0
- data/lib/cnvrg/dataset.rb +296 -0
- data/lib/cnvrg/experiment.rb +19 -10
- data/lib/cnvrg/files.rb +302 -208
- data/lib/cnvrg/helpers.rb +42 -1
- data/lib/cnvrg/job.rb +0 -1
- data/lib/cnvrg/project.rb +55 -15
- data/lib/cnvrg/ssh.rb +95 -0
- data/lib/cnvrg/version.rb +2 -1
- data/lib/cnvrg.rb +0 -1
- metadata +48 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffd4052a08f7eb71ea98aa8f19de96bdff87fc44
|
4
|
+
data.tar.gz: 3b6590a222fac4dd9f76ff426741c6a8307d03d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80a6c9f306f598d71b0b16e1cad9ada5d25183ec634197c49b4271784ffb39afc33e0d1c68b4729dcc27c426442355a29b0ff578bb3b60f23e108803d3ec5356
|
7
|
+
data.tar.gz: dd6e15721cb90947998f1e9e568e5f8a57bf0a6f8e16b30a9cb2d5e9d732b75a331bebe1e08ec202527bdb1b9a95972902611395a6f877da00024e79d4012912
|
data/cnvrg.gemspec
CHANGED
@@ -34,11 +34,15 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_runtime_dependency 'aws-sdk'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
|
-
spec.add_runtime_dependency 'logstash-logger'
|
38
|
-
spec.add_runtime_dependency 'launchy'
|
39
|
-
spec.add_runtime_dependency 'docker-api'
|
40
|
-
spec.add_runtime_dependency 'rubyzip'
|
41
|
-
spec.add_runtime_dependency 'activesupport'
|
37
|
+
spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
|
38
|
+
spec.add_runtime_dependency 'launchy', '~> 2.4'
|
39
|
+
spec.add_runtime_dependency 'docker-api', '~> 1.33'
|
40
|
+
spec.add_runtime_dependency 'rubyzip', '~> 1.2'
|
41
|
+
spec.add_runtime_dependency 'activesupport', '~> 5.0'
|
42
|
+
spec.add_runtime_dependency 'ruby-progressbar'
|
43
|
+
spec.add_runtime_dependency 'net-ssh'
|
44
|
+
|
45
|
+
|
42
46
|
|
43
47
|
|
44
48
|
end
|
data/lib/cnvrg/Images.rb
CHANGED
@@ -1,7 +1,8 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'cnvrg/files'
|
3
3
|
require 'docker'
|
4
|
-
require '
|
4
|
+
require 'net/ssh'
|
5
|
+
|
5
6
|
|
6
7
|
module Cnvrg
|
7
8
|
class Images
|
@@ -47,7 +48,6 @@ module Cnvrg
|
|
47
48
|
end
|
48
49
|
|
49
50
|
rescue => e
|
50
|
-
puts e
|
51
51
|
end
|
52
52
|
|
53
53
|
end
|
@@ -95,6 +95,73 @@ module Cnvrg
|
|
95
95
|
|
96
96
|
|
97
97
|
end
|
98
|
+
def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
|
99
|
+
response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
|
100
|
+
is_base:is_base,image_extend:image_extend,
|
101
|
+
python3:python3})
|
102
|
+
return response
|
103
|
+
end
|
104
|
+
def self.revoke_custom_new_image(owner,slug)
|
105
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
|
106
|
+
return response
|
107
|
+
end
|
108
|
+
def self.commit_custom_image(owner,slug)
|
109
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'GET')
|
110
|
+
return response
|
111
|
+
end
|
112
|
+
def self.ssh_to_machine(resp)
|
113
|
+
|
114
|
+
sts_path = resp["result"]["sts_path"]
|
115
|
+
|
116
|
+
uri = URI.parse(sts_path)
|
117
|
+
|
118
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
119
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
120
|
+
request = Net::HTTP::Get.new(sts_path)
|
121
|
+
|
122
|
+
body = ""
|
123
|
+
http_object.start do |http|
|
124
|
+
response = http.request request
|
125
|
+
body = response.read_body
|
126
|
+
end
|
127
|
+
|
128
|
+
URLcrypt::key = [body].pack('H*')
|
129
|
+
|
130
|
+
ip = URLcrypt.decrypt(resp["result"]["machine_i"])
|
131
|
+
|
132
|
+
user = URLcrypt.decrypt(resp["result"]["machine_u"])
|
133
|
+
key = URLcrypt.decrypt(resp["result"]["machine_k"])
|
134
|
+
tempssh = Tempfile.new "sshkey"
|
135
|
+
tempssh.write open(key).read
|
136
|
+
tempssh.rewind
|
137
|
+
key_path = tempssh.path
|
138
|
+
count = 0
|
139
|
+
while count < 5
|
140
|
+
|
141
|
+
begin
|
142
|
+
ssh = Net::SSH.start(ip, user=user, :keys => key_path, :timeout => 10)
|
143
|
+
if !ssh.nil?
|
144
|
+
return ssh
|
145
|
+
else
|
146
|
+
count+=1
|
147
|
+
sleep(2)
|
148
|
+
|
149
|
+
end
|
150
|
+
rescue
|
151
|
+
count+=1
|
152
|
+
sleep(2)
|
153
|
+
|
154
|
+
|
155
|
+
end
|
156
|
+
end
|
157
|
+
if tempssh
|
158
|
+
tempssh.close
|
159
|
+
tempssh.unlink
|
160
|
+
end
|
161
|
+
return false
|
162
|
+
end
|
163
|
+
|
164
|
+
|
98
165
|
|
99
166
|
def create_custom_image(new_image_name,working_dir,stored_commands)
|
100
167
|
|
@@ -174,9 +241,10 @@ module Cnvrg
|
|
174
241
|
def create_container(port=7654, is_remote=false)
|
175
242
|
begin
|
176
243
|
image_settings = {
|
177
|
-
'Image' => "#{@image_name}
|
244
|
+
'Image' => "#{@image_name}:latest",
|
178
245
|
'User' => 'ds',
|
179
|
-
'Cmd' => '/
|
246
|
+
'Cmd' => '/usr/local/cnvrg/run_ipython.sh',
|
247
|
+
'WorkingDir' => '/home/ds/notebooks',
|
180
248
|
'ExposedPorts' => {
|
181
249
|
'8888/tcp' => {},
|
182
250
|
},
|
@@ -189,9 +257,6 @@ module Cnvrg
|
|
189
257
|
},
|
190
258
|
},
|
191
259
|
}
|
192
|
-
# if !is_remote
|
193
|
-
# image_settings['HostConfig'].merge!({ 'Binds' => ["#{@working_dir}:/home/ds/notebooks"]})
|
194
|
-
# end
|
195
260
|
container = Docker::Container.create(image_settings)
|
196
261
|
container.start()
|
197
262
|
netrc = File.open(File.expand_path('~')+"/.netrc", "rb")
|
@@ -201,10 +266,6 @@ module Cnvrg
|
|
201
266
|
p = container.exec(command, tty: true)
|
202
267
|
command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.netrc"]
|
203
268
|
p = container.exec(command, tty: true)
|
204
|
-
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
205
|
-
container.exec(command, tty: true)
|
206
|
-
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
|
207
|
-
container.exec(command, tty: true)
|
208
269
|
config = File.open(File.expand_path('~')+"/.cnvrg/config.yml", "rb")
|
209
270
|
config_content = config.read
|
210
271
|
container.store_file("/home/ds/.cnvrg/config.yml", config_content)
|
@@ -227,7 +288,6 @@ module Cnvrg
|
|
227
288
|
if e.message.include? "is not running"
|
228
289
|
return create_container(port-1)
|
229
290
|
end
|
230
|
-
puts e
|
231
291
|
return false
|
232
292
|
rescue SignalException
|
233
293
|
|
@@ -247,17 +307,18 @@ module Cnvrg
|
|
247
307
|
File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
|
248
308
|
File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
|
249
309
|
rescue => e
|
250
|
-
puts e
|
251
310
|
end
|
252
311
|
|
253
312
|
|
254
313
|
end
|
255
314
|
|
256
|
-
def remote_notebook(notebook_path, instance_type, kernel)
|
315
|
+
def remote_notebook(notebook_path, instance_type, kernel,data,data_commit)
|
257
316
|
response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/remote_notebook", 'POST', {dir: notebook_path,
|
258
317
|
project_slug: @project_slug,
|
259
318
|
instance_type: instance_type,
|
260
|
-
kernel: kernel
|
319
|
+
kernel: kernel,
|
320
|
+
dataset_slug:data,
|
321
|
+
dataset_commit: data_commit})
|
261
322
|
return response
|
262
323
|
end
|
263
324
|
|
data/lib/cnvrg/api.rb
CHANGED
@@ -57,7 +57,10 @@ module Cnvrg
|
|
57
57
|
else
|
58
58
|
response
|
59
59
|
end
|
60
|
-
|
60
|
+
when 'POST'
|
61
|
+
conn.options.timeout = 420
|
62
|
+
conn.options.open_timeout =420
|
63
|
+
|
61
64
|
response = conn.post "#{endpoint_uri}/#{resource}", data
|
62
65
|
|
63
66
|
if parse_request == true
|
@@ -70,12 +73,12 @@ module Cnvrg
|
|
70
73
|
fr.headers['Auth-Token'] = @pass
|
71
74
|
fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
72
75
|
fr.headers["Content-Type"] = "multipart/form-data"
|
73
|
-
|
76
|
+
|
74
77
|
fr.request :multipart
|
75
78
|
fr.request :url_encoded
|
76
79
|
fr.adapter :net_http
|
77
80
|
end
|
78
|
-
|
81
|
+
|
79
82
|
|
80
83
|
# what if windows?
|
81
84
|
# data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
|
@@ -104,7 +107,7 @@ module Cnvrg
|
|
104
107
|
else
|
105
108
|
end
|
106
109
|
rescue => e
|
107
|
-
|
110
|
+
puts e
|
108
111
|
return nil
|
109
112
|
end
|
110
113
|
|