cnvrg 0.0.14 → 0.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +9 -5
- data/lib/cnvrg.rb +0 -1
- data/lib/cnvrg/Images.rb +112 -26
- data/lib/cnvrg/api.rb +7 -4
- data/lib/cnvrg/cli.rb +2201 -455
- data/lib/cnvrg/data.rb +72 -0
- data/lib/cnvrg/datafiles.rb +509 -0
- data/lib/cnvrg/dataset.rb +296 -0
- data/lib/cnvrg/experiment.rb +19 -10
- data/lib/cnvrg/files.rb +302 -208
- data/lib/cnvrg/helpers.rb +42 -1
- data/lib/cnvrg/job.rb +0 -1
- data/lib/cnvrg/project.rb +55 -15
- data/lib/cnvrg/ssh.rb +95 -0
- data/lib/cnvrg/version.rb +2 -1
- metadata +48 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ffd4052a08f7eb71ea98aa8f19de96bdff87fc44
|
4
|
+
data.tar.gz: 3b6590a222fac4dd9f76ff426741c6a8307d03d3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80a6c9f306f598d71b0b16e1cad9ada5d25183ec634197c49b4271784ffb39afc33e0d1c68b4729dcc27c426442355a29b0ff578bb3b60f23e108803d3ec5356
|
7
|
+
data.tar.gz: dd6e15721cb90947998f1e9e568e5f8a57bf0a6f8e16b30a9cb2d5e9d732b75a331bebe1e08ec202527bdb1b9a95972902611395a6f877da00024e79d4012912
|
data/cnvrg.gemspec
CHANGED
@@ -34,11 +34,15 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_runtime_dependency 'aws-sdk'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
|
-
spec.add_runtime_dependency 'logstash-logger'
|
38
|
-
spec.add_runtime_dependency 'launchy'
|
39
|
-
spec.add_runtime_dependency 'docker-api'
|
40
|
-
spec.add_runtime_dependency 'rubyzip'
|
41
|
-
spec.add_runtime_dependency 'activesupport'
|
37
|
+
spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
|
38
|
+
spec.add_runtime_dependency 'launchy', '~> 2.4'
|
39
|
+
spec.add_runtime_dependency 'docker-api', '~> 1.33'
|
40
|
+
spec.add_runtime_dependency 'rubyzip', '~> 1.2'
|
41
|
+
spec.add_runtime_dependency 'activesupport', '~> 5.0'
|
42
|
+
spec.add_runtime_dependency 'ruby-progressbar'
|
43
|
+
spec.add_runtime_dependency 'net-ssh'
|
44
|
+
|
45
|
+
|
42
46
|
|
43
47
|
|
44
48
|
end
|
data/lib/cnvrg.rb
CHANGED
data/lib/cnvrg/Images.rb
CHANGED
@@ -1,6 +1,8 @@
|
|
1
1
|
require 'fileutils'
|
2
2
|
require 'cnvrg/files'
|
3
3
|
require 'docker'
|
4
|
+
require 'net/ssh'
|
5
|
+
|
4
6
|
|
5
7
|
module Cnvrg
|
6
8
|
class Images
|
@@ -46,7 +48,6 @@ module Cnvrg
|
|
46
48
|
end
|
47
49
|
|
48
50
|
rescue => e
|
49
|
-
puts e
|
50
51
|
end
|
51
52
|
|
52
53
|
end
|
@@ -80,22 +81,111 @@ module Cnvrg
|
|
80
81
|
|
81
82
|
end
|
82
83
|
end
|
84
|
+
def store_image_build_commands(working_dir, cmd)
|
85
|
+
begin
|
86
|
+
custom_image_file = working_dir+"/.cnvrg/custom_image.txt"
|
87
|
+
if !File.exist? custom_image_file
|
88
|
+
FileUtils.touch [custom_image_file]
|
89
|
+
end
|
90
|
+
File.open(custom_image_file, 'a' ) do |f|
|
91
|
+
f.puts cmd
|
92
|
+
end
|
93
|
+
rescue
|
94
|
+
end
|
83
95
|
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
96
|
+
|
97
|
+
end
|
98
|
+
def self.create_new_custom_image(type,owner,image_name,is_public,is_base,image_extend,python3)
|
99
|
+
response = Cnvrg::API.request("users/#{owner}/images/custom", 'POST', {instance_type:type,image_name:image_name,is_public:is_public,
|
100
|
+
is_base:is_base,image_extend:image_extend,
|
101
|
+
python3:python3})
|
102
|
+
return response
|
103
|
+
end
|
104
|
+
def self.revoke_custom_new_image(owner,slug)
|
105
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/revoke_image", 'GET')
|
106
|
+
return response
|
107
|
+
end
|
108
|
+
def self.commit_custom_image(owner,slug)
|
109
|
+
response = Cnvrg::API.request("users/#{owner}/images/#{slug}/commit_custom_image", 'GET')
|
110
|
+
return response
|
111
|
+
end
|
112
|
+
def self.ssh_to_machine(resp)
|
113
|
+
|
114
|
+
sts_path = resp["result"]["sts_path"]
|
115
|
+
|
116
|
+
uri = URI.parse(sts_path)
|
117
|
+
|
118
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
119
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
120
|
+
request = Net::HTTP::Get.new(sts_path)
|
121
|
+
|
122
|
+
body = ""
|
123
|
+
http_object.start do |http|
|
124
|
+
response = http.request request
|
125
|
+
body = response.read_body
|
126
|
+
end
|
127
|
+
|
128
|
+
URLcrypt::key = [body].pack('H*')
|
129
|
+
|
130
|
+
ip = URLcrypt.decrypt(resp["result"]["machine_i"])
|
131
|
+
|
132
|
+
user = URLcrypt.decrypt(resp["result"]["machine_u"])
|
133
|
+
key = URLcrypt.decrypt(resp["result"]["machine_k"])
|
134
|
+
tempssh = Tempfile.new "sshkey"
|
135
|
+
tempssh.write open(key).read
|
136
|
+
tempssh.rewind
|
137
|
+
key_path = tempssh.path
|
138
|
+
count = 0
|
139
|
+
while count < 5
|
140
|
+
|
141
|
+
begin
|
142
|
+
ssh = Net::SSH.start(ip, user=user, :keys => key_path, :timeout => 10)
|
143
|
+
if !ssh.nil?
|
144
|
+
return ssh
|
145
|
+
else
|
146
|
+
count+=1
|
147
|
+
sleep(2)
|
148
|
+
|
149
|
+
end
|
150
|
+
rescue
|
151
|
+
count+=1
|
152
|
+
sleep(2)
|
153
|
+
|
154
|
+
|
155
|
+
end
|
156
|
+
end
|
157
|
+
if tempssh
|
158
|
+
tempssh.close
|
159
|
+
tempssh.unlink
|
160
|
+
end
|
161
|
+
return false
|
162
|
+
end
|
163
|
+
|
164
|
+
|
165
|
+
|
166
|
+
def create_custom_image(new_image_name,working_dir,stored_commands)
|
167
|
+
|
168
|
+
python2_arr = get_installed_packages("python2")
|
169
|
+
py2 = python2_arr.join(",") unless python2_arr.nil? or python2_arr.empty?
|
170
|
+
python3_arr = get_installed_packages("python3")
|
171
|
+
py3 = python3_arr.join(",") unless python3_arr.nil? or python3_arr.empty?
|
88
172
|
system_arr = get_installed_packages("system")
|
89
173
|
sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
|
90
|
-
|
91
|
-
response = Cnvrg::API.request("users/#{@owner}/
|
92
|
-
dpkg: sys, new_image_name: new_image_name
|
174
|
+
|
175
|
+
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/images/push", 'POST', {image_slug: @image_slug, py2: py2,py3:py3,
|
176
|
+
dpkg: sys, new_image_name: new_image_name,
|
177
|
+
run_commands:stored_commands})
|
93
178
|
if Cnvrg::CLI.is_response_success(response) and !response["result"]["slug"].nil?
|
94
179
|
container = get_container()
|
95
|
-
|
180
|
+
name = response["result"]["name"]
|
181
|
+
container = get_container()
|
182
|
+
container.commit({repo:name,tag:"latest"})
|
183
|
+
update_image(name+":latest", container, response["result"]["slug"])
|
184
|
+
File.truncate(working_dir+"/.cnvrg/custom_image.txt", 0)
|
185
|
+
|
96
186
|
end
|
97
187
|
|
98
|
-
return
|
188
|
+
return true
|
99
189
|
|
100
190
|
end
|
101
191
|
|
@@ -151,9 +241,10 @@ module Cnvrg
|
|
151
241
|
def create_container(port=7654, is_remote=false)
|
152
242
|
begin
|
153
243
|
image_settings = {
|
154
|
-
'Image' => "#{@image_name}
|
244
|
+
'Image' => "#{@image_name}:latest",
|
155
245
|
'User' => 'ds',
|
156
|
-
'Cmd' => '/
|
246
|
+
'Cmd' => '/usr/local/cnvrg/run_ipython.sh',
|
247
|
+
'WorkingDir' => '/home/ds/notebooks',
|
157
248
|
'ExposedPorts' => {
|
158
249
|
'8888/tcp' => {},
|
159
250
|
},
|
@@ -166,9 +257,6 @@ module Cnvrg
|
|
166
257
|
},
|
167
258
|
},
|
168
259
|
}
|
169
|
-
# if !is_remote
|
170
|
-
# image_settings['HostConfig'].merge!({ 'Binds' => ["#{@working_dir}:/home/ds/notebooks"]})
|
171
|
-
# end
|
172
260
|
container = Docker::Container.create(image_settings)
|
173
261
|
container.start()
|
174
262
|
netrc = File.open(File.expand_path('~')+"/.netrc", "rb")
|
@@ -178,10 +266,6 @@ module Cnvrg
|
|
178
266
|
p = container.exec(command, tty: true)
|
179
267
|
command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.netrc"]
|
180
268
|
p = container.exec(command, tty: true)
|
181
|
-
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
182
|
-
container.exec(command, tty: true)
|
183
|
-
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
|
184
|
-
container.exec(command, tty: true)
|
185
269
|
config = File.open(File.expand_path('~')+"/.cnvrg/config.yml", "rb")
|
186
270
|
config_content = config.read
|
187
271
|
container.store_file("/home/ds/.cnvrg/config.yml", config_content)
|
@@ -204,7 +288,6 @@ module Cnvrg
|
|
204
288
|
if e.message.include? "is not running"
|
205
289
|
return create_container(port-1)
|
206
290
|
end
|
207
|
-
puts e
|
208
291
|
return false
|
209
292
|
rescue SignalException
|
210
293
|
|
@@ -224,25 +307,28 @@ module Cnvrg
|
|
224
307
|
File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
|
225
308
|
File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
|
226
309
|
rescue => e
|
227
|
-
puts e
|
228
310
|
end
|
229
311
|
|
230
312
|
|
231
313
|
end
|
232
314
|
|
233
|
-
def remote_notebook(notebook_path, instance_type, kernel)
|
315
|
+
def remote_notebook(notebook_path, instance_type, kernel,data,data_commit)
|
234
316
|
response = Cnvrg::API.request("users/#{@owner}/images/#{@image_slug}/remote_notebook", 'POST', {dir: notebook_path,
|
235
317
|
project_slug: @project_slug,
|
236
318
|
instance_type: instance_type,
|
237
|
-
kernel: kernel
|
319
|
+
kernel: kernel,
|
320
|
+
dataset_slug:data,
|
321
|
+
dataset_commit: data_commit})
|
238
322
|
return response
|
239
323
|
end
|
240
324
|
|
241
325
|
def get_installed_packages(repo)
|
242
326
|
container = get_container()
|
243
327
|
case repo
|
244
|
-
when "
|
328
|
+
when "python2"
|
245
329
|
command = ['/bin/bash', '-lc', '/opt/ds/bin/pip freeze']
|
330
|
+
when "python3"
|
331
|
+
command = ['/bin/bash', '-lc', '/opt/ds3/bin/pip3 freeze']
|
246
332
|
when "system"
|
247
333
|
command = ["/bin/bash", "-lc", "dpkg -l | grep '^ii' | awk '{print $2\"==\"$3}'"]
|
248
334
|
end
|
@@ -269,8 +355,8 @@ module Cnvrg
|
|
269
355
|
py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
|
270
356
|
system_arr = self.get_installed_packages("system")
|
271
357
|
sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
|
272
|
-
bash_history = self.get_bash_history
|
273
|
-
diff = [py, sys
|
358
|
+
# bash_history = self.get_bash_history
|
359
|
+
diff = [py, sys]
|
274
360
|
|
275
361
|
end
|
276
362
|
|
data/lib/cnvrg/api.rb
CHANGED
@@ -57,7 +57,10 @@ module Cnvrg
|
|
57
57
|
else
|
58
58
|
response
|
59
59
|
end
|
60
|
-
|
60
|
+
when 'POST'
|
61
|
+
conn.options.timeout = 420
|
62
|
+
conn.options.open_timeout =420
|
63
|
+
|
61
64
|
response = conn.post "#{endpoint_uri}/#{resource}", data
|
62
65
|
|
63
66
|
if parse_request == true
|
@@ -70,12 +73,12 @@ module Cnvrg
|
|
70
73
|
fr.headers['Auth-Token'] = @pass
|
71
74
|
fr.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
72
75
|
fr.headers["Content-Type"] = "multipart/form-data"
|
73
|
-
|
76
|
+
|
74
77
|
fr.request :multipart
|
75
78
|
fr.request :url_encoded
|
76
79
|
fr.adapter :net_http
|
77
80
|
end
|
78
|
-
|
81
|
+
|
79
82
|
|
80
83
|
# what if windows?
|
81
84
|
# data[:file] = Faraday::UploadIO.new(data[:absolute_path], content_type)
|
@@ -104,7 +107,7 @@ module Cnvrg
|
|
104
107
|
else
|
105
108
|
end
|
106
109
|
rescue => e
|
107
|
-
|
110
|
+
puts e
|
108
111
|
return nil
|
109
112
|
end
|
110
113
|
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -7,7 +7,7 @@ require 'uri'
|
|
7
7
|
require 'open-uri'
|
8
8
|
require 'json'
|
9
9
|
require 'yaml'
|
10
|
-
require 'digest' #
|
10
|
+
require 'digest' # sha1up
|
11
11
|
require "highline/import"
|
12
12
|
require 'socket'
|
13
13
|
include Open4
|
@@ -18,6 +18,10 @@ require 'cnvrg/project'
|
|
18
18
|
require 'cnvrg/files'
|
19
19
|
require 'cnvrg/experiment'
|
20
20
|
require 'cnvrg/Images'
|
21
|
+
require 'cnvrg/dataset'
|
22
|
+
require 'cnvrg/datafiles'
|
23
|
+
require 'cnvrg/data'
|
24
|
+
require 'cnvrg/ssh'
|
21
25
|
require 'etc'
|
22
26
|
require 'logstash-logger'
|
23
27
|
require 'cnvrg/job'
|
@@ -29,27 +33,116 @@ require 'fileutils'
|
|
29
33
|
require 'zip'
|
30
34
|
require 'active_support/all'
|
31
35
|
require 'thor'
|
36
|
+
require 'pathname'
|
37
|
+
require 'enumerator'
|
38
|
+
|
39
|
+
class Thor
|
40
|
+
module Base
|
41
|
+
def initialize(args = [], local_options = {}, config = {})
|
42
|
+
parse_options = Thor.class_options
|
43
|
+
|
44
|
+
# The start method splits inbound arguments at the first argument
|
45
|
+
# that looks like an option (starts with - or --). It then calls
|
46
|
+
# new, passing in the two halves of the arguments Array as the
|
47
|
+
# first two parameters.
|
48
|
+
|
49
|
+
command_options = config.delete(:command_options) # hook for start
|
50
|
+
parse_options = parse_options.merge(command_options) if command_options
|
51
|
+
if local_options.is_a?(Array)
|
52
|
+
array_options = local_options
|
53
|
+
hash_options = {}
|
54
|
+
else
|
55
|
+
# Handle the case where the class was explicitly instantiated
|
56
|
+
# with pre-parsed options.
|
57
|
+
array_options = []
|
58
|
+
hash_options = local_options
|
59
|
+
|
60
|
+
end
|
61
|
+
|
62
|
+
# Let Thor::Options parse the options first, so it can remove
|
63
|
+
# declared options from the array. This will leave us with
|
64
|
+
# a list of arguments that weren't declared.
|
65
|
+
stop_on_unknown = Thor.stop_on_unknown_option? config[:current_command]
|
66
|
+
opts = Thor::Options.new(parse_options, hash_options, stop_on_unknown)
|
67
|
+
real_options = []
|
68
|
+
|
69
|
+
real_args = [].replace(array_options)
|
70
|
+
if local_options.is_a? (Array) and !local_options.empty? and args.empty?
|
71
|
+
array_options.each_with_index do |p, i|
|
72
|
+
opt = p
|
73
|
+
if p.include? "="
|
74
|
+
opt = p.split("=")[0]
|
75
|
+
end
|
76
|
+
option = is_option(parse_options.values, opt)
|
77
|
+
if !option
|
78
|
+
break
|
79
|
+
else
|
80
|
+
real_options << p
|
81
|
+
real_args.delete(p)
|
82
|
+
if !p.include? "=" and option.type != :boolean
|
83
|
+
if i+1< array_options.size
|
84
|
+
real_options << array_options[i+1]
|
85
|
+
real_args.delete(array_options[i+1])
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
end
|
92
|
+
|
93
|
+
args = real_args
|
94
|
+
else
|
95
|
+
if !args.empty? and local_options.is_a? Array and !local_options.empty?
|
96
|
+
args = args + local_options
|
97
|
+
else
|
98
|
+
args = args.flatten()
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
102
|
+
|
103
|
+
|
104
|
+
self.options = opts.parse(real_options)
|
105
|
+
self.options = config[:class_options].merge(options) if config[:class_options]
|
106
|
+
|
107
|
+
# If unknown options are disallowed, make sure that none of the
|
108
|
+
# remaining arguments looks like an option.
|
109
|
+
opts.check_unknown! if Thor.check_unknown_options?(config)
|
110
|
+
|
111
|
+
# Add the remaining arguments from the options parser to the
|
112
|
+
# arguments passed in to initialize. Then remove any positional
|
113
|
+
# arguments declared using #argument (this is primarily used
|
114
|
+
# by Thor::Group). Tis will leave us with the remaining
|
115
|
+
# positional arguments.
|
116
|
+
to_parse = args
|
117
|
+
to_parse += opts.remaining unless self.class.strict_args_position?(config)
|
118
|
+
thor_args = Thor::Arguments.new(self.class.arguments)
|
119
|
+
thor_args.parse(to_parse).each { |k, v| __send__("#{k}=", v) }
|
120
|
+
@args = thor_args.remaining
|
121
|
+
end
|
122
|
+
|
123
|
+
def is_option (options, p)
|
124
|
+
options.each do |o|
|
125
|
+
if !o.aliases.nil?
|
126
|
+
if (o.aliases.is_a? Array and o.aliases.include? p) or (o.aliases.is_a? Array and o.aliases.size ==1 and o.aliases[0].split(",").include? p) or o.switch_name.eql? p
|
127
|
+
return o
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
end
|
132
|
+
return false
|
133
|
+
end
|
134
|
+
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
32
138
|
|
33
|
-
# DEV VERSION
|
34
|
-
#
|
35
139
|
module Cnvrg
|
140
|
+
|
36
141
|
class CLI < Thor
|
37
142
|
|
38
143
|
INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
|
39
144
|
IP="localhost"
|
40
145
|
PORT=7654
|
41
|
-
desc '', ''
|
42
|
-
|
43
|
-
def printable_commands(all = true, subcommand = false)
|
44
|
-
(all ? all_commands : commands).map do |_, command|
|
45
|
-
next if command.hidden? or (command.description.empty? and command.usage.empty?)
|
46
|
-
item = []
|
47
|
-
item << banner(command, false, subcommand)
|
48
|
-
item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
|
49
|
-
item
|
50
|
-
end.compact
|
51
|
-
end
|
52
|
-
|
53
146
|
class << self
|
54
147
|
# Hackery.Take the run method away from Thor so that we can redefine it.
|
55
148
|
def is_thor_reserved_word?(word, type)
|
@@ -57,9 +150,10 @@ module Cnvrg
|
|
57
150
|
super
|
58
151
|
end
|
59
152
|
end
|
153
|
+
desc "data", "upload and manage datasets", :hide => true
|
154
|
+
subcommand "data", Data
|
60
155
|
|
61
|
-
desc "", ""
|
62
|
-
method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
|
156
|
+
desc "", "", :hide => true
|
63
157
|
|
64
158
|
def test
|
65
159
|
# image_settings = {
|
@@ -77,11 +171,10 @@ module Cnvrg
|
|
77
171
|
# },
|
78
172
|
# },
|
79
173
|
# }
|
80
|
-
container = Docker::Container.get('b4d64bf83f41')
|
81
|
-
s = "/leah/1/2/3/4/5"
|
82
|
-
command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
83
|
-
puts container.exec(command, tty: true)
|
84
|
-
|
174
|
+
# container = Docker::Container.get('b4d64bf83f41')
|
175
|
+
# s = "/leah/1/2/3/4/5"
|
176
|
+
# command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
177
|
+
# puts container.exec(command, tty: true)
|
85
178
|
end
|
86
179
|
|
87
180
|
|
@@ -94,7 +187,7 @@ module Cnvrg
|
|
94
187
|
|
95
188
|
map %w(-v --version) => :version
|
96
189
|
|
97
|
-
desc 'set api url
|
190
|
+
desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
|
98
191
|
|
99
192
|
def set_api_url(url)
|
100
193
|
home_dir = File.expand_path('~')
|
@@ -110,15 +203,19 @@ module Cnvrg
|
|
110
203
|
end
|
111
204
|
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
112
205
|
owner = config.to_h[:owner]
|
206
|
+
compression_path = "#{File.expand_path('~')}/.cnvrg/tmp"
|
113
207
|
|
114
208
|
say "Setting default api to be: #{url}", Thor::Shell::Color::BLUE
|
115
209
|
if config.empty?
|
116
|
-
config = {owner: "", username: "", version_last_check: get_start_day(), api: url}
|
210
|
+
config = {owner: "", username: "", version_last_check: get_start_day(), api: url,compression_path: compression_path }
|
117
211
|
else
|
118
|
-
|
212
|
+
if !config.to_h[:compression_path].nil?
|
213
|
+
compression_path = config.to_h[:compression_path]
|
214
|
+
end
|
215
|
+
config = {owner: config.to_h[:owner], username: config.to_h[:username], version_last_check: config.to_h[:version_last_check], api: url, compression_path: compression_path}
|
119
216
|
end
|
120
217
|
res = Cnvrg::API.request("/users/#{owner}/custom_api", 'POST', {custom_api: url})
|
121
|
-
if Cnvrg::CLI.is_response_success(res)
|
218
|
+
if Cnvrg::CLI.is_response_success(res,false)
|
122
219
|
|
123
220
|
checks = Helpers.checkmark
|
124
221
|
|
@@ -136,7 +233,7 @@ module Cnvrg
|
|
136
233
|
end
|
137
234
|
end
|
138
235
|
|
139
|
-
desc '', ''
|
236
|
+
desc '', '', :hide => true
|
140
237
|
|
141
238
|
def set_remote_api_url(owner, current_user, url)
|
142
239
|
home_dir = File.expand_path('~')
|
@@ -151,7 +248,9 @@ module Cnvrg
|
|
151
248
|
FileUtils.touch [home_dir+"/.cnvrg/config.yml"]
|
152
249
|
end
|
153
250
|
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
154
|
-
|
251
|
+
|
252
|
+
compression_path = "#{home_dir}/.cnvrg/tmp"
|
253
|
+
config = {owner: owner, username: current_user, version_last_check: get_start_day(), api: url,compression_path:compression_path}
|
155
254
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
156
255
|
say "Done"
|
157
256
|
rescue
|
@@ -161,57 +260,108 @@ module Cnvrg
|
|
161
260
|
|
162
261
|
map %w(-api --api) => :set_api_url
|
163
262
|
|
164
|
-
desc '
|
263
|
+
desc 'set_default_owner', 'set default owner'
|
165
264
|
|
166
265
|
def set_default_owner
|
266
|
+
begin
|
267
|
+
path = File.expand_path('~')+"/.cnvrg/config.yml"
|
268
|
+
if !File.exist?(path)
|
269
|
+
say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
|
167
270
|
|
168
|
-
|
169
|
-
|
170
|
-
|
271
|
+
exit(0)
|
272
|
+
end
|
273
|
+
config = YAML.load_file(path)
|
171
274
|
|
172
|
-
|
173
|
-
|
174
|
-
|
275
|
+
username = config.to_h[:username]
|
276
|
+
res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
|
277
|
+
if Cnvrg::CLI.is_response_success(res)
|
278
|
+
owner = username
|
279
|
+
result = res["result"]
|
280
|
+
owners = result["owners"]
|
281
|
+
urls = result["urls"]
|
282
|
+
choose_owner = result["username"]
|
283
|
+
if owners.empty?
|
284
|
+
else
|
285
|
+
owners << choose_owner
|
286
|
+
chosen = false
|
287
|
+
while !chosen
|
288
|
+
owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
|
289
|
+
choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
|
290
|
+
|
291
|
+
if choose_owner =~ /[[:digit:]]/
|
292
|
+
ow_index = choose_owner.to_i-1
|
293
|
+
if ow_index<0 or ow_index >= owners.size
|
294
|
+
say "No such owner, please choose again", Thor::Shell::Color::BLUE
|
295
|
+
chosen = false
|
296
|
+
next
|
297
|
+
end
|
298
|
+
choose_owner = owners[choose_owner.to_i-1]
|
299
|
+
chosen = true
|
175
300
|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
301
|
+
else
|
302
|
+
|
303
|
+
owners_lower = owners.map { |o| o.downcase }
|
304
|
+
ow_index = owners_lower.index(choose_owner.downcase)
|
305
|
+
if ow_index.nil?
|
306
|
+
say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
|
307
|
+
else
|
308
|
+
chosen = true
|
309
|
+
end
|
310
|
+
end
|
184
311
|
|
185
|
-
if owners.empty?
|
186
|
-
else
|
187
|
-
owners << choose_owner
|
188
|
-
chosen = false
|
189
|
-
while !chosen
|
190
|
-
choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
|
191
|
-
|
192
|
-
owners_lower = owners.map { |o| o.downcase }
|
193
|
-
ow_index = owners_lower.index(choose_owner.downcase)
|
194
|
-
if ow_index.nil?
|
195
|
-
say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
|
196
|
-
else
|
197
|
-
chosen = true
|
198
312
|
end
|
313
|
+
|
314
|
+
|
315
|
+
end
|
316
|
+
if set_owner(choose_owner, result["username"], urls[ow_index])
|
317
|
+
say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
|
318
|
+
else
|
319
|
+
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
199
320
|
end
|
321
|
+
end
|
322
|
+
rescue SignalException
|
323
|
+
say "\nAborting"
|
324
|
+
exit(1)
|
325
|
+
end
|
326
|
+
end
|
200
327
|
|
328
|
+
desc 'set_compression_path', 'set compression path'
|
329
|
+
method_option :reset, :type => :boolean, :aliases => ["-r","--reset"], :default => false
|
201
330
|
|
331
|
+
def set_compression_path(*compression_path)
|
332
|
+
begin
|
333
|
+
if (compression_path.nil? or compression_path.empty?) and options["reset"]
|
334
|
+
compression_path = ["#{File.expand_path('~')}/.cnvrg/tmp"]
|
202
335
|
end
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
336
|
+
compression_path = compression_path.join(" ")
|
337
|
+
if !Dir.exist? compression_path
|
338
|
+
say "Couldn't find #{compression_path}, please make sure it exist", Thor::Shell::Color::RED
|
339
|
+
exit(0)
|
207
340
|
end
|
341
|
+
|
342
|
+
home_dir = File.expand_path('~')
|
343
|
+
path = "#{home_dir}/.cnvrg/config.yml"
|
344
|
+
if !File.exist?(path)
|
345
|
+
say "Couldn't find ~/.cnvrg/config.yml file, please logout and login again", Thor::Shell::Color::RED
|
346
|
+
|
347
|
+
exit(0)
|
348
|
+
end
|
349
|
+
config = YAML.load_file(path)
|
350
|
+
config_new = {owner: config.to_h[:owner], username: config.to_h[:username],
|
351
|
+
version_last_check: config.to_h[:version_last_check], api: config.to_h[:api], compression_path: compression_path}
|
352
|
+
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config_new.to_yaml }
|
353
|
+
checks = Helpers.checkmark
|
354
|
+
say "#{checks} Done", Thor::Shell::Color::GREEN
|
355
|
+
|
356
|
+
rescue SignalException
|
357
|
+
say "\nAborting"
|
358
|
+
exit(1)
|
208
359
|
end
|
209
360
|
end
|
210
361
|
|
211
|
-
map %w(--set-default-owner) => :set_default_owner
|
212
362
|
|
213
363
|
|
214
|
-
desc 'login', 'Authenticate with cnvrg.io
|
364
|
+
desc 'login', 'Authenticate with cnvrg.io platform'
|
215
365
|
|
216
366
|
def login
|
217
367
|
begin
|
@@ -239,25 +389,46 @@ module Cnvrg
|
|
239
389
|
owners = result["owners"]
|
240
390
|
urls = result["urls"]
|
241
391
|
choose_owner = result["username"]
|
242
|
-
|
392
|
+
ow_index = 0
|
243
393
|
if owners.empty?
|
394
|
+
choose_owner = result["username"]
|
244
395
|
else
|
245
|
-
|
246
|
-
|
247
|
-
while !chosen
|
248
|
-
choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
|
396
|
+
choose_owner = owners[0]
|
397
|
+
end
|
249
398
|
|
250
|
-
|
251
|
-
|
252
|
-
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
399
|
+
# if owners.empty?
|
400
|
+
# else
|
401
|
+
# owners << choose_owner
|
402
|
+
# chosen = false
|
403
|
+
# while !chosen
|
404
|
+
# owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
|
405
|
+
# choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
|
406
|
+
#
|
407
|
+
# if choose_owner =~ /[[:digit:]]/
|
408
|
+
# ow_index = choose_owner.to_i-1
|
409
|
+
# if ow_index<0 or ow_index >= owners.size
|
410
|
+
# say "No such owner, please choose again", Thor::Shell::Color::BLUE
|
411
|
+
# chosen = false
|
412
|
+
# next
|
413
|
+
# end
|
414
|
+
# choose_owner = owners[choose_owner.to_i-1]
|
415
|
+
# chosen = true
|
416
|
+
#
|
417
|
+
# else
|
418
|
+
#
|
419
|
+
# owners_lower = owners.map { |o| o.downcase }
|
420
|
+
# ow_index = owners_lower.index(choose_owner.downcase)
|
421
|
+
# if ow_index.nil?
|
422
|
+
# say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
|
423
|
+
# else
|
424
|
+
# chosen = true
|
425
|
+
# end
|
426
|
+
# end
|
427
|
+
#
|
428
|
+
# end
|
258
429
|
|
259
430
|
|
260
|
-
end
|
431
|
+
# end
|
261
432
|
|
262
433
|
if set_owner(choose_owner, result["username"], urls[ow_index])
|
263
434
|
say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
|
@@ -273,6 +444,11 @@ module Cnvrg
|
|
273
444
|
|
274
445
|
exit(1)
|
275
446
|
end
|
447
|
+
rescue
|
448
|
+
|
449
|
+
say "Error Occurred, aborting", Thor::Shell::Color::RED
|
450
|
+
logout()
|
451
|
+
exit(1)
|
276
452
|
rescue SignalException
|
277
453
|
|
278
454
|
say "\nAborting"
|
@@ -302,7 +478,6 @@ module Cnvrg
|
|
302
478
|
|
303
479
|
def me()
|
304
480
|
begin
|
305
|
-
|
306
481
|
verify_logged_in(false)
|
307
482
|
log_start(__method__, args, options)
|
308
483
|
auth = Cnvrg::Auth.new
|
@@ -313,6 +488,7 @@ module Cnvrg
|
|
313
488
|
end
|
314
489
|
|
315
490
|
log_end(0)
|
491
|
+
|
316
492
|
rescue SignalException
|
317
493
|
log_end(-1)
|
318
494
|
|
@@ -323,8 +499,8 @@ module Cnvrg
|
|
323
499
|
|
324
500
|
## Projects
|
325
501
|
desc 'new', 'Create a new cnvrg project'
|
326
|
-
method_option :clean, :type => :boolean, :aliases => ["-c"
|
327
|
-
method_option :docker_image, :type => :string, :aliases => ["-d"
|
502
|
+
method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
|
503
|
+
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
328
504
|
|
329
505
|
def new(project_name)
|
330
506
|
begin
|
@@ -354,7 +530,7 @@ module Cnvrg
|
|
354
530
|
if docker_image_local.size == 0
|
355
531
|
|
356
532
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
357
|
-
image =
|
533
|
+
image = pull(docker_image)
|
358
534
|
if image
|
359
535
|
say "downloaded image: #{docker_image}"
|
360
536
|
@image = Images.new(working_dir, docker_image)
|
@@ -392,8 +568,8 @@ module Cnvrg
|
|
392
568
|
say "created project successfully", Thor::Shell::Color::GREEN
|
393
569
|
say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
|
394
570
|
rescue => e
|
395
|
-
|
396
|
-
|
571
|
+
log_end(-1, e.message)
|
572
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
397
573
|
if Dir.exist? working_dir
|
398
574
|
|
399
575
|
@project.revert(working_dir)
|
@@ -411,6 +587,7 @@ module Cnvrg
|
|
411
587
|
exit(1)
|
412
588
|
end
|
413
589
|
end
|
590
|
+
|
414
591
|
desc 'set_image', 'set_image for a project'
|
415
592
|
|
416
593
|
def set_image(docker_image)
|
@@ -422,7 +599,7 @@ module Cnvrg
|
|
422
599
|
if docker_image_local.size == 0
|
423
600
|
|
424
601
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
425
|
-
image =
|
602
|
+
image = pull(docker_image)
|
426
603
|
if image
|
427
604
|
say "downloaded image: #{docker_image}"
|
428
605
|
@image = Images.new(working_dir, docker_image)
|
@@ -447,9 +624,10 @@ module Cnvrg
|
|
447
624
|
@image.update_image_activity(nil, nil)
|
448
625
|
|
449
626
|
end
|
450
|
-
|
451
|
-
|
452
|
-
method_option :
|
627
|
+
|
628
|
+
desc 'link', 'Link current directory to a new cnvrg project'
|
629
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
630
|
+
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
453
631
|
|
454
632
|
def link
|
455
633
|
begin
|
@@ -473,7 +651,7 @@ module Cnvrg
|
|
473
651
|
end
|
474
652
|
working_dir = Dir.getwd
|
475
653
|
owner = CLI.get_owner
|
476
|
-
if Project.link(owner,project_name, docker)
|
654
|
+
if Project.link(owner, project_name, docker)
|
477
655
|
path = Dir.pwd
|
478
656
|
@project = Project.new(path)
|
479
657
|
@project.generate_idx()
|
@@ -482,7 +660,7 @@ module Cnvrg
|
|
482
660
|
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.include? docker_image }.flatten
|
483
661
|
if docker_image_local.size == 0
|
484
662
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
485
|
-
image =
|
663
|
+
image = pull(docker_image)
|
486
664
|
if image
|
487
665
|
say "downloaded image: #{docker_image}"
|
488
666
|
@image = Images.new(working_dir, docker_image)
|
@@ -506,40 +684,728 @@ module Cnvrg
|
|
506
684
|
@image = Images.new(working_dir, image_name)
|
507
685
|
end
|
508
686
|
end
|
509
|
-
if sync
|
510
|
-
say "Syncing project", Thor::Shell::Color::BLUE
|
511
|
-
upload(true)
|
687
|
+
if sync
|
688
|
+
say "Syncing project", Thor::Shell::Color::BLUE
|
689
|
+
upload(true)
|
690
|
+
end
|
691
|
+
|
692
|
+
url = @project.url
|
693
|
+
check = Helpers.checkmark
|
694
|
+
say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
|
695
|
+
say "#{project_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
|
696
|
+
log_end(0)
|
697
|
+
|
698
|
+
else
|
699
|
+
log_end(1, "can't link project")
|
700
|
+
@project.revert(working_dir) unless @project.nil?
|
701
|
+
say "Error linking project, please contact support.", Thor::Shell::Color::RED
|
702
|
+
exit(0)
|
703
|
+
end
|
704
|
+
rescue SignalException
|
705
|
+
log_end(-1)
|
706
|
+
|
707
|
+
say "\nAborting"
|
708
|
+
exit(1)
|
709
|
+
end
|
710
|
+
end
|
711
|
+
|
712
|
+
desc 'data init', 'Init dataset directory'
|
713
|
+
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
714
|
+
|
715
|
+
def init_data(public)
|
716
|
+
begin
|
717
|
+
verify_logged_in(false)
|
718
|
+
log_start(__method__, args, options)
|
719
|
+
dataset_name =File.basename(Dir.getwd)
|
720
|
+
if File.directory?(Dir.getwd+"/.cnvrg")
|
721
|
+
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
722
|
+
say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
|
723
|
+
|
724
|
+
exit(0)
|
725
|
+
end
|
726
|
+
say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
|
727
|
+
|
728
|
+
working_dir = Dir.getwd
|
729
|
+
owner = CLI.get_owner
|
730
|
+
if Dataset.init(owner, dataset_name, options["public"])
|
731
|
+
path = Dir.pwd
|
732
|
+
@dataset = Dataset.new(path)
|
733
|
+
|
734
|
+
url = @dataset.url
|
735
|
+
check = Helpers.checkmark
|
736
|
+
say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
|
737
|
+
say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
|
738
|
+
log_end(0)
|
739
|
+
|
740
|
+
else
|
741
|
+
log_end(1, "can't create dataset")
|
742
|
+
@dataset.revert(working_dir) unless @dataset.nil?
|
743
|
+
say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
|
744
|
+
exit(0)
|
745
|
+
end
|
746
|
+
rescue SignalException
|
747
|
+
log_end(-1)
|
748
|
+
|
749
|
+
say "\nAborting"
|
750
|
+
exit(1)
|
751
|
+
end
|
752
|
+
end
|
753
|
+
|
754
|
+
desc 'data clone', 'Clone dataset'
|
755
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
756
|
+
|
757
|
+
def clone_data(dataset_url)
|
758
|
+
begin
|
759
|
+
verify_logged_in(false)
|
760
|
+
log_start(__method__, args, options)
|
761
|
+
url_parts = dataset_url.split("/")
|
762
|
+
project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
|
763
|
+
slug = url_parts[project_index+1]
|
764
|
+
owner = url_parts[project_index-1]
|
765
|
+
response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
|
766
|
+
|
767
|
+
Cnvrg::CLI.is_response_success(response)
|
768
|
+
dataset_name = response["result"]["name"]
|
769
|
+
if (Dir.exists? dataset_name)
|
770
|
+
say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
|
771
|
+
if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
772
|
+
say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
|
773
|
+
log_end(1, "conflict with dir #{dataset_name}")
|
774
|
+
|
775
|
+
exit(1)
|
776
|
+
end
|
777
|
+
|
778
|
+
end
|
779
|
+
if Dataset.clone(owner, dataset_name, slug)
|
780
|
+
say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
|
781
|
+
|
782
|
+
commit_to_clone = options["commit"] || nil
|
783
|
+
working_dir = "#{Dir.pwd}/#{dataset_name}"
|
784
|
+
@dataset = Dataset.new(working_dir)
|
785
|
+
@dataset.generate_idx()
|
786
|
+
say "Downloading data", Thor::Shell::Color::BLUE
|
787
|
+
|
788
|
+
|
789
|
+
download_data(false, false, path = working_dir, in_dir=false)
|
790
|
+
|
791
|
+
|
792
|
+
check = Helpers.checkmark
|
793
|
+
say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
|
794
|
+
log_end(0)
|
795
|
+
|
796
|
+
else
|
797
|
+
log_end(1, "can't create dataset")
|
798
|
+
@dataset.revert(working_dir) unless @dataset.nil?
|
799
|
+
say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
|
800
|
+
exit(0)
|
801
|
+
end
|
802
|
+
rescue SignalException
|
803
|
+
log_end(-1)
|
804
|
+
|
805
|
+
say "\nAborting"
|
806
|
+
exit(1)
|
807
|
+
end
|
808
|
+
end
|
809
|
+
|
810
|
+
desc 'init_data_container', 'Init dataset directory', :hide => true
|
811
|
+
method_option :login_content, :type => :string, :aliases => ["-l"], :default => ""
|
812
|
+
|
813
|
+
def init_data_container(container)
|
814
|
+
begin
|
815
|
+
login_content = options["login_content"]
|
816
|
+
|
817
|
+
container = Docker::Container.get(container)
|
818
|
+
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
819
|
+
container.exec(command, tty: true)
|
820
|
+
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
821
|
+
container.exec(command, tty: true)
|
822
|
+
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
|
823
|
+
container.exec(command, tty: true)
|
824
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
|
825
|
+
container.exec(command, tty: true)
|
826
|
+
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
827
|
+
container.exec(command, tty: true)
|
828
|
+
|
829
|
+
rescue SignalException
|
830
|
+
log_end(-1)
|
831
|
+
|
832
|
+
say "\nAborting"
|
833
|
+
exit(1)
|
834
|
+
end
|
835
|
+
end
|
836
|
+
|
837
|
+
desc 'data_snap', 'Init dataset directory', :hide => true
|
838
|
+
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
839
|
+
|
840
|
+
def snap_data
|
841
|
+
begin
|
842
|
+
verify_logged_in(false)
|
843
|
+
log_start(__method__, args, options)
|
844
|
+
|
845
|
+
owner = CLI.get_owner
|
846
|
+
path = Dir.pwd
|
847
|
+
@dataset = Dataset.new(path)
|
848
|
+
|
849
|
+
log_end(0)
|
850
|
+
|
851
|
+
rescue SignalException
|
852
|
+
log_end(-1)
|
853
|
+
|
854
|
+
say "\nAborting"
|
855
|
+
exit(1)
|
856
|
+
end
|
857
|
+
end
|
858
|
+
|
859
|
+
desc 'data_snap', 'Init dataset directory', :hide => true
|
860
|
+
|
861
|
+
def data_init_container(owner, dataset_slug, dataset_name)
|
862
|
+
|
863
|
+
if Dataset.init_container(owner, dataset_slug, dataset_name)
|
864
|
+
|
865
|
+
say "init finished successfully", Thor::Shell::Color::GREEN
|
866
|
+
log_end(0)
|
867
|
+
|
868
|
+
else
|
869
|
+
log_end(1, "can't create dataset")
|
870
|
+
say "error creating dataset, please contact support.", Thor::Shell::Color::RED
|
871
|
+
exit(0)
|
872
|
+
end
|
873
|
+
end
|
874
|
+
|
875
|
+
desc 'data download', 'pull data'
|
876
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
877
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
878
|
+
|
879
|
+
def download_data(verbose, sync, path=Dir.pwd, in_dir=true)
|
880
|
+
begin
|
881
|
+
verify_logged_in(in_dir)
|
882
|
+
log_start(__method__, args, options)
|
883
|
+
if path.nil? or path.empty?
|
884
|
+
path = Dir.pwd
|
885
|
+
end
|
886
|
+
dataset_dir = is_cnvrg_dir(path)
|
887
|
+
@dataset = Dataset.new(dataset_dir)
|
888
|
+
|
889
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
890
|
+
new_branch = options["new_branch"] || false
|
891
|
+
|
892
|
+
res = @dataset.compare_idx(new_branch)["result"]
|
893
|
+
|
894
|
+
result = res["tree"]
|
895
|
+
|
896
|
+
commit = res["commit"]
|
897
|
+
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
898
|
+
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
899
|
+
log_end(0)
|
900
|
+
return true
|
901
|
+
end
|
902
|
+
result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
|
903
|
+
|
904
|
+
delete = result["result"]["delete"]
|
905
|
+
commits = result["result"]["commits"]
|
906
|
+
updated_idx = result["result"]["idx"]
|
907
|
+
commits.each do |c|
|
908
|
+
file_name = @files.download_data_file(c, dataset_dir)
|
909
|
+
|
910
|
+
if file_name.eql? false or file_name.nil?
|
911
|
+
##RETRY
|
912
|
+
next
|
913
|
+
end
|
914
|
+
file_path = "#{dataset_dir}/#{file_name}"
|
915
|
+
success = extarct_tar(file_path, dataset_dir)
|
916
|
+
if !success
|
917
|
+
#TODO: ?
|
918
|
+
end
|
919
|
+
|
920
|
+
FileUtils.rm_rf([file_path])
|
921
|
+
|
922
|
+
end
|
923
|
+
to_delete = []
|
924
|
+
delete.each do |d|
|
925
|
+
to_delete << "#{dataset_dir}/#{d}"
|
926
|
+
end
|
927
|
+
FileUtils.rm_rf(to_delete)
|
928
|
+
|
929
|
+
@dataset.update_idx(updated_idx)
|
930
|
+
|
931
|
+
|
932
|
+
# result["conflicts"].each do |f|
|
933
|
+
# relative_path = f.gsub(/^#{@dataset.local_path}/, "")
|
934
|
+
# if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
|
935
|
+
# successful_changes << relative_path
|
936
|
+
# end
|
937
|
+
#
|
938
|
+
# end
|
939
|
+
# update idx with latest commit
|
940
|
+
# @dataset.update_idx_with_commit!(commit)
|
941
|
+
check = Helpers.checkmark()
|
942
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
943
|
+
|
944
|
+
|
945
|
+
log_end(0)
|
946
|
+
end
|
947
|
+
rescue
|
948
|
+
log_end(-1)
|
949
|
+
|
950
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
951
|
+
exit(1)
|
952
|
+
rescue SignalException
|
953
|
+
log_end(-1)
|
954
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
955
|
+
exit(1)
|
956
|
+
end
|
957
|
+
|
958
|
+
|
959
|
+
desc 'upload_data', 'Upload data files', :hide => true
|
960
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
961
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
|
962
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
963
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
964
|
+
|
965
|
+
def upload_data(sync=false, direct=false)
|
966
|
+
|
967
|
+
begin
|
968
|
+
verify_logged_in(true)
|
969
|
+
log_start(__method__, args, options)
|
970
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
971
|
+
@dataset = Dataset.new(dataset_dir)
|
972
|
+
|
973
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
974
|
+
ignore = options[:ignore] || []
|
975
|
+
if !@dataset.update_ignore_list(ignore)
|
976
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
977
|
+
end
|
978
|
+
result = @dataset.compare_idx(false)
|
979
|
+
|
980
|
+
commit = result["result"]["commit"]
|
981
|
+
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
982
|
+
log_end(0)
|
983
|
+
|
984
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
985
|
+
exit(1)
|
986
|
+
end
|
987
|
+
|
988
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
989
|
+
result = result["result"]["tree"]
|
990
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
991
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
992
|
+
# if all.size == 1
|
993
|
+
# num = "conflict"
|
994
|
+
# else
|
995
|
+
# num = "conflicts"
|
996
|
+
# end
|
997
|
+
# say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
|
998
|
+
# say "#{all.join("\n")}"
|
999
|
+
# say "Please fix #{num}, and retry", Thor::Shell::Color::RED
|
1000
|
+
# exit(1)
|
1001
|
+
#
|
1002
|
+
# end
|
1003
|
+
check = Helpers.checkmark()
|
1004
|
+
|
1005
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1006
|
+
log_end(0)
|
1007
|
+
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
1008
|
+
return true
|
1009
|
+
end
|
1010
|
+
update_count = 0
|
1011
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
1012
|
+
successful_updates = []
|
1013
|
+
successful_deletions = []
|
1014
|
+
if options["verbose"]
|
1015
|
+
if update_total == 1
|
1016
|
+
say "Updating #{update_total} file", Thor::Shell::Color::BLUE
|
1017
|
+
else
|
1018
|
+
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
1019
|
+
end
|
1020
|
+
else
|
1021
|
+
say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
|
1022
|
+
|
1023
|
+
end
|
1024
|
+
|
1025
|
+
# Start commit
|
1026
|
+
|
1027
|
+
commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
|
1028
|
+
# upload / update
|
1029
|
+
begin
|
1030
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
1031
|
+
absolute_path = "#{@dataset.local_path}/#{f}"
|
1032
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1033
|
+
if File.directory?(absolute_path)
|
1034
|
+
resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
|
1035
|
+
if resDir
|
1036
|
+
update_count += 1
|
1037
|
+
successful_updates<< relative_path
|
1038
|
+
end
|
1039
|
+
else
|
1040
|
+
res = @files.upload_file(absolute_path, relative_path, commit_sha1)
|
1041
|
+
|
1042
|
+
if res
|
1043
|
+
update_count += 1
|
1044
|
+
successful_updates<< relative_path
|
1045
|
+
else
|
1046
|
+
@files.rollback_commit(commit_sha1)
|
1047
|
+
log_end(1, "can't upload, Rolling Back all changes")
|
1048
|
+
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
1049
|
+
exit(0)
|
1050
|
+
end
|
1051
|
+
end
|
1052
|
+
end
|
1053
|
+
|
1054
|
+
# delete
|
1055
|
+
deleted = update_deleted(result["deleted"])
|
1056
|
+
deleted.each do |f|
|
1057
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1058
|
+
if relative_path.end_with?("/")
|
1059
|
+
if @files.delete_dir(f, relative_path, commit_sha1)
|
1060
|
+
# update_count += 1
|
1061
|
+
successful_updates<< relative_path
|
1062
|
+
end
|
1063
|
+
else
|
1064
|
+
if @files.delete_file(f, relative_path, commit_sha1)
|
1065
|
+
# update_count += 1
|
1066
|
+
successful_updates<< relative_path
|
1067
|
+
end
|
1068
|
+
end
|
1069
|
+
end
|
1070
|
+
|
1071
|
+
rescue SignalException
|
1072
|
+
log_end(-1)
|
1073
|
+
@files.rollback_commit(commit_sha1)
|
1074
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1075
|
+
exit(0)
|
1076
|
+
rescue => e
|
1077
|
+
log_end(-1, e.message)
|
1078
|
+
@files.rollback_commit(commit_sha1)
|
1079
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
1080
|
+
exit(0)
|
1081
|
+
end
|
1082
|
+
if !result["deleted"].nil? and !result["deleted"].empty?
|
1083
|
+
update_count += result["deleted"].size
|
1084
|
+
end
|
1085
|
+
if update_count == update_total
|
1086
|
+
res = @files.end_commit(commit_sha1)
|
1087
|
+
if (Cnvrg::CLI.is_response_success(res, false))
|
1088
|
+
# save idx
|
1089
|
+
begin
|
1090
|
+
@dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
1091
|
+
|
1092
|
+
@dataset.update_idx_with_commit!(commit_sha1)
|
1093
|
+
rescue => e
|
1094
|
+
log_end(-1, e.message)
|
1095
|
+
@files.rollback_commit(commit_sha1)
|
1096
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
1097
|
+
exit(1)
|
1098
|
+
|
1099
|
+
end
|
1100
|
+
if options["verbose"]
|
1101
|
+
say "#{check} Done", Thor::Shell::Color::BLUE
|
1102
|
+
if successful_updates.size >0
|
1103
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
1104
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1105
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
1106
|
+
end
|
1107
|
+
if successful_deletions.size >0
|
1108
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
1109
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1110
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
1111
|
+
end
|
1112
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
1113
|
+
else
|
1114
|
+
if (options["sync"] or sync) and direct
|
1115
|
+
say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
|
1116
|
+
|
1117
|
+
else
|
1118
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
1119
|
+
|
1120
|
+
end
|
1121
|
+
|
1122
|
+
end
|
1123
|
+
|
1124
|
+
log_end(0)
|
1125
|
+
else
|
1126
|
+
@files.rollback_commit(commit_sha1)
|
1127
|
+
log_end(1, "error. Rolling Back all changes")
|
1128
|
+
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
1129
|
+
end
|
1130
|
+
else
|
1131
|
+
log_end(1, "error. Rolling Back all changes")
|
1132
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
1133
|
+
|
1134
|
+
@files.rollback_commit(commit_sha1)
|
1135
|
+
end
|
1136
|
+
rescue => e
|
1137
|
+
|
1138
|
+
log_end(-1)
|
1139
|
+
|
1140
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
1141
|
+
@files.rollback_commit(commit_sha1)
|
1142
|
+
exit(1)
|
1143
|
+
rescue SignalException
|
1144
|
+
log_end(-1)
|
1145
|
+
|
1146
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
1147
|
+
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
1148
|
+
@files.rollback_commit(commit_sha1)
|
1149
|
+
exit(1)
|
1150
|
+
end
|
1151
|
+
|
1152
|
+
end
|
1153
|
+
|
1154
|
+
desc 'data upload', 'push data'
|
1155
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
1156
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1157
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1158
|
+
method_option :no_compression, :type => :boolean, :aliases => ["-nc", "--no_compression"], :default => false
|
1159
|
+
|
1160
|
+
def upload_data_tar(ignore, verbose, sync,no_compression)
|
1161
|
+
|
1162
|
+
begin
|
1163
|
+
verify_logged_in(true)
|
1164
|
+
log_start(__method__, args, options)
|
1165
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1166
|
+
|
1167
|
+
@dataset = Dataset.new(dataset_dir)
|
1168
|
+
|
1169
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
1170
|
+
if !@dataset.update_ignore_list(ignore)
|
1171
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::RED
|
1172
|
+
exit(1)
|
1173
|
+
end
|
1174
|
+
say "Checking dataset", Thor::Shell::Color::BLUE
|
1175
|
+
local_idx = @dataset.generate_idx
|
1176
|
+
result = @dataset.compare_idx(false, commit=@dataset.last_local_commit, local_idx= local_idx)
|
1177
|
+
|
1178
|
+
|
1179
|
+
commit = result["result"]["commit"]
|
1180
|
+
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
1181
|
+
log_end(0)
|
1182
|
+
|
1183
|
+
say "Remote server has an updated version, please run `cnvrg data download` first", Thor::Shell::Color::YELLOW
|
1184
|
+
exit(1)
|
1185
|
+
end
|
1186
|
+
|
1187
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
|
1188
|
+
result = result["result"]["tree"]
|
1189
|
+
check = Helpers.checkmark()
|
1190
|
+
|
1191
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1192
|
+
log_end(0)
|
1193
|
+
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
|
1194
|
+
return true
|
1195
|
+
end
|
1196
|
+
update_count = 0
|
1197
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
1198
|
+
successful_updates = []
|
1199
|
+
successful_deletions = []
|
1200
|
+
|
1201
|
+
# Start commit
|
1202
|
+
res = @files.start_commit(false)["result"]
|
1203
|
+
commit_sha1 =res["commit_sha1"]
|
1204
|
+
commit_time = res["commit_time"]
|
1205
|
+
# upload / update
|
1206
|
+
begin
|
1207
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
1208
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1209
|
+
successful_updates<< relative_path
|
1210
|
+
update_count += 1
|
1211
|
+
end
|
1212
|
+
|
1213
|
+
# delete
|
1214
|
+
deleted = update_deleted(result["deleted"])
|
1215
|
+
deleted.each do |f|
|
1216
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1217
|
+
successful_updates<< relative_path
|
1218
|
+
end
|
1219
|
+
@dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
|
1220
|
+
|
1221
|
+
@dataset.update_idx_with_commit!(commit_sha1)
|
1222
|
+
say "Compressing data", Thor::Shell::Color::BLUE
|
1223
|
+
|
1224
|
+
home_dir = File.expand_path('~')
|
1225
|
+
compression_path = get_compression_path
|
1226
|
+
tar_path = "#{compression_path}#{@dataset.slug}_#{commit_sha1}.tar.gz"
|
1227
|
+
tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
|
1228
|
+
tar_files = (result["added"] + result["updated_on_local"]).join("\n")
|
1229
|
+
File.open(tar_files_path, 'w') { |f| f.write tar_files }
|
1230
|
+
is_tar = create_tar(dataset_dir, tar_path, tar_files_path,no_compression)
|
1231
|
+
if !is_tar
|
1232
|
+
say "ERROR: Couldn't compress data", Thor::Shell::Color::RED
|
1233
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1234
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1235
|
+
@files.rollback_commit(commit_sha1)
|
1236
|
+
say "Rolling Back all changes.", Thor::Shell::Color::RED
|
1237
|
+
exit(1)
|
1238
|
+
end
|
1239
|
+
say "Uploading data", Thor::Shell::Color::BLUE
|
1240
|
+
res = false
|
1241
|
+
res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
|
1242
|
+
|
1243
|
+
if res
|
1244
|
+
say "Commiting data", Thor::Shell::Color::BLUE
|
1245
|
+
|
1246
|
+
cur_idx = @dataset.get_idx.to_h
|
1247
|
+
|
1248
|
+
res = @files.end_commit_tar(commit_sha1, cur_idx)
|
1249
|
+
if !Cnvrg::CLI.is_response_success(res, false)
|
1250
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1251
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1252
|
+
|
1253
|
+
|
1254
|
+
@files.rollback_commit(commit_sha1)
|
1255
|
+
say "Can't commit, Rolling Back all changes.", Thor::Shell::Color::RED
|
1256
|
+
exit(1)
|
1257
|
+
end
|
1258
|
+
|
1259
|
+
else
|
1260
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1261
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1262
|
+
|
1263
|
+
|
1264
|
+
@files.rollback_commit(commit_sha1)
|
1265
|
+
say "Can't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
1266
|
+
exit(1)
|
1267
|
+
end
|
1268
|
+
|
1269
|
+
|
1270
|
+
# delete
|
1271
|
+
FileUtils.rm_rf([tar_path, tar_files_path])
|
1272
|
+
|
1273
|
+
rescue SignalException
|
1274
|
+
log_end(-1)
|
1275
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1276
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1277
|
+
|
1278
|
+
|
1279
|
+
@files.rollback_commit(commit_sha1)
|
1280
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1281
|
+
exit(0)
|
1282
|
+
rescue => e
|
1283
|
+
puts e.message
|
1284
|
+
if !Cnvrg::Helpers.internet_connection?
|
1285
|
+
say "Seems there is no internet connection", Thor::Shell::Color::RED
|
1286
|
+
|
512
1287
|
end
|
1288
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1289
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
513
1290
|
|
514
|
-
|
515
|
-
|
516
|
-
say "
|
517
|
-
|
518
|
-
|
1291
|
+
log_end(-1, e.message)
|
1292
|
+
@files.rollback_commit(commit_sha1)
|
1293
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
1294
|
+
exit(0)
|
1295
|
+
end
|
1296
|
+
if verbose
|
1297
|
+
say "#{check} Done", Thor::Shell::Color::BLUE
|
1298
|
+
if successful_updates.size >0
|
1299
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
1300
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1301
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
1302
|
+
end
|
1303
|
+
if successful_deletions.size >0
|
1304
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
1305
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1306
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
1307
|
+
end
|
519
1308
|
|
1309
|
+
|
1310
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
520
1311
|
else
|
521
|
-
|
522
|
-
|
523
|
-
say "Error linking project, please contact support.", Thor::Shell::Color::RED
|
524
|
-
exit(0)
|
1312
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
1313
|
+
|
525
1314
|
end
|
1315
|
+
|
1316
|
+
|
1317
|
+
rescue => e
|
1318
|
+
log_end(-1, e.message)
|
1319
|
+
|
1320
|
+
|
1321
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
1322
|
+
@files.rollback_commit(commit_sha1)
|
1323
|
+
exit(1)
|
526
1324
|
rescue SignalException
|
527
1325
|
log_end(-1)
|
528
1326
|
|
529
|
-
say "\nAborting"
|
1327
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
1328
|
+
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
1329
|
+
@files.rollback_commit(commit_sha1)
|
530
1330
|
exit(1)
|
531
1331
|
end
|
1332
|
+
|
1333
|
+
log_end(0)
|
1334
|
+
|
1335
|
+
end
|
1336
|
+
|
1337
|
+
|
1338
|
+
desc 'unlink', 'Unlink a project from current directory', :hide => true
|
1339
|
+
|
1340
|
+
def create_volume
|
1341
|
+
verify_logged_in(false)
|
1342
|
+
log_start(__method__, args, options)
|
1343
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1344
|
+
@dataset = Dataset.new(dataset_dir)
|
1345
|
+
@dataset.create_volume()
|
1346
|
+
|
1347
|
+
end
|
1348
|
+
|
1349
|
+
desc 'data list', 'List all dataset you currently have'
|
1350
|
+
|
1351
|
+
def list_dataset
|
1352
|
+
verify_logged_in(false)
|
1353
|
+
log_start(__method__, args, options)
|
1354
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1355
|
+
@dataset = Dataset.new(dataset_dir)
|
1356
|
+
owner = @dataset.owner
|
1357
|
+
if owner.nil? or owner.empty?
|
1358
|
+
owner = CLI.get_owner()
|
1359
|
+
end
|
1360
|
+
result = @dataset.list(owner)
|
1361
|
+
list = result["result"]["list"]
|
1362
|
+
|
1363
|
+
print_table(list)
|
1364
|
+
|
1365
|
+
end
|
1366
|
+
|
1367
|
+
desc 'data commits', 'List all commits for a specific dataset'
|
1368
|
+
|
1369
|
+
def list_dataset_commits()
|
1370
|
+
verify_logged_in(true)
|
1371
|
+
log_start(__method__, args, options)
|
1372
|
+
|
1373
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1374
|
+
@dataset = Dataset.new(dataset_dir)
|
1375
|
+
result = @dataset.list_commits()
|
1376
|
+
list = result["result"]["list"]
|
1377
|
+
|
1378
|
+
print_table(list)
|
1379
|
+
|
1380
|
+
end
|
1381
|
+
|
1382
|
+
desc 'commits', 'List all commits for a specific dataset'
|
1383
|
+
|
1384
|
+
def list_commits()
|
1385
|
+
verify_logged_in(true)
|
1386
|
+
log_start(__method__, args, options)
|
1387
|
+
|
1388
|
+
project_dir = is_cnvrg_dir(Dir.pwd)
|
1389
|
+
@project = Project.new(project_dir)
|
1390
|
+
result = @project.list_commits()
|
1391
|
+
list = result["result"]["list"]
|
1392
|
+
|
1393
|
+
print_table(list)
|
1394
|
+
|
532
1395
|
end
|
533
|
-
|
1396
|
+
|
1397
|
+
|
1398
|
+
desc 'unlink', 'Unlink a project from current directory'
|
1399
|
+
|
534
1400
|
def unlink
|
535
1401
|
verify_logged_in(false)
|
536
1402
|
log_start(__method__, args, options)
|
537
1403
|
working_dir = is_cnvrg_dir()
|
538
|
-
list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
|
1404
|
+
list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
|
539
1405
|
FileUtils.rm_rf list_to_del
|
540
1406
|
end
|
541
1407
|
|
542
|
-
desc 'clone', 'Clone
|
1408
|
+
desc 'clone', 'Clone project'
|
543
1409
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
|
544
1410
|
method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
|
545
1411
|
|
@@ -587,6 +1453,7 @@ module Cnvrg
|
|
587
1453
|
Cnvrg::CLI.is_response_success response
|
588
1454
|
working_dir = project_home
|
589
1455
|
docker_image = response["result"]["image"]
|
1456
|
+
current_commit = response["result"]["commit"]
|
590
1457
|
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
591
1458
|
File.open(working_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
592
1459
|
if !docker_image.nil? and !docker_image.empty? and !remote
|
@@ -594,7 +1461,7 @@ module Cnvrg
|
|
594
1461
|
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{docker_image}:latest" }.flatten
|
595
1462
|
if docker_image_local.size == 0
|
596
1463
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
597
|
-
image =
|
1464
|
+
image = pull(docker_image)
|
598
1465
|
if image
|
599
1466
|
say "downloaded image: #{docker_image}"
|
600
1467
|
@image = Images.new(working_dir, docker_image)
|
@@ -628,7 +1495,8 @@ module Cnvrg
|
|
628
1495
|
end
|
629
1496
|
else
|
630
1497
|
# blob
|
631
|
-
|
1498
|
+
|
1499
|
+
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=current_commit)
|
632
1500
|
successful_changes << relative_path
|
633
1501
|
end
|
634
1502
|
end
|
@@ -667,6 +1535,7 @@ module Cnvrg
|
|
667
1535
|
commit = result["commit"]
|
668
1536
|
result = result["tree"]
|
669
1537
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
1538
|
+
|
670
1539
|
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
671
1540
|
say "Project is up to date", Thor::Shell::Color::GREEN
|
672
1541
|
log_end(0)
|
@@ -713,14 +1582,36 @@ module Cnvrg
|
|
713
1582
|
end
|
714
1583
|
end
|
715
1584
|
|
1585
|
+
desc '', '', :hide => true
|
1586
|
+
|
1587
|
+
def revert_exp
|
1588
|
+
begin
|
1589
|
+
log_start(__method__, args, options)
|
1590
|
+
@project = Project.new(get_project_home)
|
1591
|
+
|
1592
|
+
result = @project.compare_idx(false)["result"]
|
1593
|
+
result = result["tree"]
|
1594
|
+
if result["added"].size > 0
|
1595
|
+
FileUtils.rm_rf(result["added"])
|
1596
|
+
end
|
1597
|
+
say "Changes were removed successfully", Thor::Shell::Color::GREEN
|
1598
|
+
|
1599
|
+
|
1600
|
+
rescue SignalException
|
1601
|
+
log_end(-1)
|
1602
|
+
say "\nAborting"
|
1603
|
+
exit(1)
|
1604
|
+
end
|
1605
|
+
end
|
1606
|
+
|
716
1607
|
|
717
1608
|
desc 'upload', 'Upload updated files'
|
718
|
-
method_option :ignore, :type => :
|
719
|
-
method_option :new_branch, :type => :boolean, :aliases => ["-nb"
|
720
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
721
|
-
method_option :sync, :type => :boolean, :aliases => ["
|
1609
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
1610
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1611
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1612
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
722
1613
|
|
723
|
-
def upload(link=false, sync=false)
|
1614
|
+
def upload(link=false, sync=false, direct=false, ignore_list="")
|
724
1615
|
|
725
1616
|
begin
|
726
1617
|
verify_logged_in(true)
|
@@ -729,12 +1620,30 @@ module Cnvrg
|
|
729
1620
|
@project = Project.new(get_project_home)
|
730
1621
|
|
731
1622
|
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
732
|
-
ignore = options[:ignore] ||
|
1623
|
+
ignore = options[:ignore] || ""
|
1624
|
+
|
1625
|
+
if ignore.nil? or ignore.empty?
|
1626
|
+
ignore = ignore_list
|
1627
|
+
end
|
1628
|
+
data_ignore = data_dir_include()
|
1629
|
+
if !data_ignore.nil?
|
1630
|
+
if ignore.nil? or ignore.empty?
|
1631
|
+
ignore = data_ignore
|
1632
|
+
else
|
1633
|
+
ignore ="#{ignore},#{data_ignore}"
|
1634
|
+
end
|
1635
|
+
end
|
733
1636
|
if !@project.update_ignore_list(ignore)
|
734
1637
|
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
735
1638
|
end
|
736
1639
|
new_branch = options["new_branch"] || false
|
737
1640
|
|
1641
|
+
if options["sync"] or sync
|
1642
|
+
new_branch_exp = @project.get_new_branch
|
1643
|
+
if new_branch_exp
|
1644
|
+
new_branch = new_branch_exp
|
1645
|
+
end
|
1646
|
+
end
|
738
1647
|
result = @project.compare_idx(new_branch)
|
739
1648
|
commit = result["result"]["commit"]
|
740
1649
|
if !link
|
@@ -748,9 +1657,23 @@ module Cnvrg
|
|
748
1657
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
749
1658
|
end
|
750
1659
|
result = result["result"]["tree"]
|
1660
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
1661
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
1662
|
+
# if all.size == 1
|
1663
|
+
# num = "conflict"
|
1664
|
+
# else
|
1665
|
+
# num = "conflicts"
|
1666
|
+
# end
|
1667
|
+
# say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
|
1668
|
+
# say "#{all.join("\n")}"
|
1669
|
+
# say "Please fix #{num}, and retry", Thor::Shell::Color::RED
|
1670
|
+
# exit(1)
|
1671
|
+
#
|
1672
|
+
# end
|
1673
|
+
check = Helpers.checkmark()
|
751
1674
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
752
1675
|
log_end(0)
|
753
|
-
say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
|
1676
|
+
say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
754
1677
|
return true
|
755
1678
|
end
|
756
1679
|
update_count = 0
|
@@ -764,7 +1687,7 @@ module Cnvrg
|
|
764
1687
|
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
765
1688
|
end
|
766
1689
|
else
|
767
|
-
say "Syncing files", Thor::Shell::Color::BLUE unless options["sync"]
|
1690
|
+
say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
|
768
1691
|
|
769
1692
|
end
|
770
1693
|
|
@@ -798,21 +1721,21 @@ module Cnvrg
|
|
798
1721
|
end
|
799
1722
|
|
800
1723
|
# delete
|
801
|
-
result["deleted"]
|
1724
|
+
deleted = update_deleted(result["deleted"])
|
1725
|
+
deleted.each do |f|
|
802
1726
|
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
803
1727
|
if relative_path.end_with?("/")
|
804
1728
|
if @files.delete_dir(f, relative_path, commit_sha1)
|
805
|
-
update_count += 1
|
1729
|
+
# update_count += 1
|
806
1730
|
successful_updates<< relative_path
|
807
1731
|
end
|
808
1732
|
else
|
809
1733
|
if @files.delete_file(f, relative_path, commit_sha1)
|
810
|
-
update_count += 1
|
1734
|
+
# update_count += 1
|
811
1735
|
successful_updates<< relative_path
|
812
1736
|
end
|
813
1737
|
end
|
814
1738
|
end
|
815
|
-
log_end(0)
|
816
1739
|
|
817
1740
|
rescue SignalException
|
818
1741
|
log_end(-1)
|
@@ -820,11 +1743,14 @@ module Cnvrg
|
|
820
1743
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
821
1744
|
exit(0)
|
822
1745
|
rescue => e
|
823
|
-
log_end(1, e.message)
|
1746
|
+
log_end(-1, e.message)
|
824
1747
|
@files.rollback_commit(commit_sha1)
|
825
1748
|
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
826
1749
|
exit(0)
|
827
1750
|
end
|
1751
|
+
if !result["deleted"].nil? and !result["deleted"].empty?
|
1752
|
+
update_count += result["deleted"].size
|
1753
|
+
end
|
828
1754
|
if update_count == update_total
|
829
1755
|
res = @files.end_commit(commit_sha1)
|
830
1756
|
if (Cnvrg::CLI.is_response_success(res, false))
|
@@ -834,7 +1760,7 @@ module Cnvrg
|
|
834
1760
|
|
835
1761
|
@project.update_idx_with_commit!(commit_sha1)
|
836
1762
|
rescue => e
|
837
|
-
log_end(1, e.message)
|
1763
|
+
log_end(-1, e.message)
|
838
1764
|
@files.rollback_commit(commit_sha1)
|
839
1765
|
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
840
1766
|
exit(1)
|
@@ -844,7 +1770,6 @@ module Cnvrg
|
|
844
1770
|
if image and image.is_docker
|
845
1771
|
image.update_image_activity(commit_sha1, nil)
|
846
1772
|
end
|
847
|
-
check = Helpers.checkmark()
|
848
1773
|
|
849
1774
|
if options["verbose"]
|
850
1775
|
say "#{check} Done", Thor::Shell::Color::BLUE
|
@@ -860,7 +1785,13 @@ module Cnvrg
|
|
860
1785
|
end
|
861
1786
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
862
1787
|
else
|
863
|
-
|
1788
|
+
if (options["sync"] or sync) and direct
|
1789
|
+
say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
1790
|
+
|
1791
|
+
else
|
1792
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
1793
|
+
|
1794
|
+
end
|
864
1795
|
|
865
1796
|
end
|
866
1797
|
|
@@ -872,13 +1803,14 @@ module Cnvrg
|
|
872
1803
|
end
|
873
1804
|
else
|
874
1805
|
log_end(1, "error. Rolling Back all changes")
|
1806
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
875
1807
|
|
876
1808
|
@files.rollback_commit(commit_sha1)
|
877
1809
|
end
|
878
|
-
rescue
|
1810
|
+
rescue => e
|
879
1811
|
log_end(-1)
|
880
1812
|
|
881
|
-
say "Error occurd, \nAborting", Thor::Shell::Color::
|
1813
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
882
1814
|
@files.rollback_commit(commit_sha1)
|
883
1815
|
exit(1)
|
884
1816
|
rescue SignalException
|
@@ -893,27 +1825,57 @@ module Cnvrg
|
|
893
1825
|
end
|
894
1826
|
|
895
1827
|
desc 'download', 'Download updated files'
|
896
|
-
method_option :new_branch, :type => :boolean, :aliases => ["-nb"
|
897
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
898
|
-
method_option :sync, :type => :boolean, :aliases => ["
|
1828
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1829
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1830
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1831
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
899
1832
|
|
900
|
-
def download
|
1833
|
+
def download(sync=false, ignore_list="")
|
901
1834
|
begin
|
902
1835
|
verify_logged_in(true)
|
903
1836
|
log_start(__method__, args, options)
|
904
1837
|
project_home = get_project_home
|
905
1838
|
@project = Project.new(project_home)
|
906
1839
|
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
1840
|
+
ignore = options[:ignore] || ""
|
1841
|
+
if ignore.nil? or ignore.empty?
|
1842
|
+
ignore = ignore_list
|
1843
|
+
end
|
1844
|
+
data_ignore = data_dir_include()
|
1845
|
+
if !data_ignore.nil?
|
1846
|
+
if ignore.nil? or ignore.empty?
|
1847
|
+
ignore = data_ignore
|
1848
|
+
else
|
1849
|
+
ignore ="#{ignore},#{data_ignore}"
|
1850
|
+
end
|
1851
|
+
end
|
1852
|
+
if !@project.update_ignore_list(ignore)
|
1853
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
1854
|
+
end
|
907
1855
|
new_branch = options["new_branch"] || false
|
908
1856
|
|
909
1857
|
res = @project.compare_idx(new_branch)["result"]
|
1858
|
+
|
910
1859
|
result = res["tree"]
|
1860
|
+
|
911
1861
|
commit = res["commit"]
|
912
|
-
if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
|
913
|
-
say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
|
1862
|
+
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
1863
|
+
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
914
1864
|
log_end(0)
|
915
1865
|
return true
|
916
1866
|
end
|
1867
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
1868
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
1869
|
+
# if all.size == 1
|
1870
|
+
# num = "1 conflict"
|
1871
|
+
# else
|
1872
|
+
# num = "#{result["conflicts"].size} conflicts"
|
1873
|
+
# end
|
1874
|
+
# say "Project contains #{num}:", Thor::Shell::Color::RED
|
1875
|
+
# say "#{all.join("\n")}"
|
1876
|
+
# say "Please fix them, and retry", Thor::Shell::Color::RED
|
1877
|
+
# exit(1)
|
1878
|
+
# end
|
917
1879
|
update_count = 0
|
918
1880
|
update_total = result["updated_on_server"].size + result["conflicts"].size
|
919
1881
|
|
@@ -935,7 +1897,7 @@ module Cnvrg
|
|
935
1897
|
|
936
1898
|
result["conflicts"].each do |f|
|
937
1899
|
relative_path = f.gsub(/^#{@project.local_path}/, "")
|
938
|
-
if @files.download_file_s3(f, relative_path, project_home, conflict=true)
|
1900
|
+
if @files.download_file_s3(f, relative_path, project_home, commit_sha1=nil, conflict=true)
|
939
1901
|
successful_changes << relative_path
|
940
1902
|
end
|
941
1903
|
|
@@ -966,12 +1928,35 @@ module Cnvrg
|
|
966
1928
|
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
967
1929
|
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
968
1930
|
else
|
969
|
-
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
1931
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
|
970
1932
|
end
|
971
1933
|
|
972
1934
|
|
973
1935
|
log_end(0)
|
974
1936
|
end
|
1937
|
+
rescue => e
|
1938
|
+
log_end(-1)
|
1939
|
+
|
1940
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
1941
|
+
if successful_changes.nil?
|
1942
|
+
exit(1)
|
1943
|
+
end
|
1944
|
+
successful_changes.each do |f|
|
1945
|
+
|
1946
|
+
abs_path = "#{@project.local_path}/#{f}"
|
1947
|
+
filename = File.basename abs_path
|
1948
|
+
say "revoking #{filename}"
|
1949
|
+
if result["conflicts"].include? f
|
1950
|
+
@files.revoke_download_file(abs_path, f, filename, true)
|
1951
|
+
elsif result["updated_on_server"].include? f
|
1952
|
+
if File.directory? abs_path
|
1953
|
+
@files.revoke_download_dir(abs_path, f, project_home)
|
1954
|
+
else
|
1955
|
+
@files.revoke_download_file(project_home, abs_path, filename)
|
1956
|
+
end
|
1957
|
+
end
|
1958
|
+
end
|
1959
|
+
exit(1)
|
975
1960
|
rescue SignalException
|
976
1961
|
log_end(-1)
|
977
1962
|
say "\nAborting", Thor::Shell::Color::BLUE
|
@@ -1025,7 +2010,7 @@ module Cnvrg
|
|
1025
2010
|
if !response["result"]["tree"].nil?
|
1026
2011
|
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
1027
2012
|
File.open(project_home + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
1028
|
-
current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
|
2013
|
+
current_tree = Dir.entries(".").reject { |file| file.start_with? '.' or file.eql? "__init__.py" or file.eql? "uwsgi.ini" }
|
1029
2014
|
|
1030
2015
|
response["result"]["tree"].each do |f|
|
1031
2016
|
relative_path = f[0].gsub(/^#{@project.local_path}/, "")
|
@@ -1037,7 +2022,7 @@ module Cnvrg
|
|
1037
2022
|
end
|
1038
2023
|
else
|
1039
2024
|
# blob
|
1040
|
-
if @files.download_file_s3(f[0], relative_path, project_home)
|
2025
|
+
if @files.download_file_s3(f[0], relative_path, project_home, commit_sha1=commit_sha1)
|
1041
2026
|
current_tree.delete(relative_path)
|
1042
2027
|
|
1043
2028
|
successful_changes << relative_path
|
@@ -1055,43 +2040,100 @@ module Cnvrg
|
|
1055
2040
|
end
|
1056
2041
|
|
1057
2042
|
|
2043
|
+
desc 'data_jump', 'jump to specific commit', :hide => true
|
2044
|
+
|
2045
|
+
def data_jump(*commit_sha1)
|
2046
|
+
begin
|
2047
|
+
verify_logged_in()
|
2048
|
+
log_start(__method__, args, options)
|
2049
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
2050
|
+
@dataset = Dataset.new(dataset_dir)
|
2051
|
+
|
2052
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
2053
|
+
if commit_sha1.nil? or commit_sha1.empty?
|
2054
|
+
commit_sha1 = @dataset.last_local_commit
|
2055
|
+
end
|
2056
|
+
response = @dataset.compare_commits(commit_sha1)
|
2057
|
+
successful_changes = []
|
2058
|
+
if !response["result"]["status"].nil?
|
2059
|
+
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
2060
|
+
File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
2061
|
+
status = response["result"]["status"]
|
2062
|
+
(status["delete"]).each do |f|
|
2063
|
+
relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
|
2064
|
+
FileUtils.rm_rf(relative_path)
|
2065
|
+
end
|
2066
|
+
# current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
|
2067
|
+
(status["dirs"]).each do |f|
|
2068
|
+
relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
|
2069
|
+
# dir
|
2070
|
+
if @files.download_dir(dataset_dir, relative_path)
|
2071
|
+
# current_tree.delete(relative_path[0, relative_path.size-1])
|
2072
|
+
successful_changes << relative_path
|
2073
|
+
end
|
2074
|
+
end
|
2075
|
+
(status["download"]).each do |f|
|
2076
|
+
relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
|
2077
|
+
# dir
|
2078
|
+
if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
|
2079
|
+
successful_changes << relative_path
|
2080
|
+
end
|
2081
|
+
end
|
2082
|
+
|
2083
|
+
|
2084
|
+
say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
|
2085
|
+
log_end(0)
|
2086
|
+
end
|
2087
|
+
|
2088
|
+
rescue SignalException
|
2089
|
+
log_end(-1)
|
2090
|
+
exi(1)
|
2091
|
+
end
|
2092
|
+
end
|
2093
|
+
|
1058
2094
|
desc 'sync', 'Sync with remote server'
|
1059
|
-
method_option :new_branch, :type => :boolean, :aliases => ["
|
1060
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
2095
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
2096
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
2097
|
+
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
|
1061
2098
|
|
1062
|
-
def sync
|
2099
|
+
def sync(direct=true)
|
2100
|
+
verify_logged_in(true) if direct
|
1063
2101
|
if options["verbose"]
|
1064
|
-
|
2102
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
1065
2103
|
else
|
1066
2104
|
say 'Syncing project', Thor::Shell::Color::BLUE
|
1067
2105
|
end
|
1068
2106
|
|
1069
2107
|
|
1070
|
-
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
|
1071
|
-
invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"]
|
1072
|
-
|
2108
|
+
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
|
2109
|
+
invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
|
2110
|
+
:ignore => options[:ignore]
|
1073
2111
|
|
1074
|
-
say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
1075
2112
|
|
1076
2113
|
end
|
1077
2114
|
|
1078
2115
|
|
1079
2116
|
desc 'run cmd', 'Runs an experiment'
|
1080
|
-
method_option :local, :type => :boolean, :aliases => ["
|
1081
|
-
method_option :small, :type => :boolean, :aliases => ["
|
1082
|
-
method_option :medium, :type => :boolean, :aliases => ["--
|
1083
|
-
method_option :large, :type => :boolean, :aliases => ["
|
2117
|
+
method_option :local, :type => :boolean, :aliases => ["-l", "--local"], :default => false
|
2118
|
+
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
2119
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
2120
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
1084
2121
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1085
|
-
method_option :gpuxl, :type => :boolean, :aliases => ["--
|
1086
|
-
method_option :
|
1087
|
-
method_option :
|
1088
|
-
method_option :
|
2122
|
+
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
2123
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
2124
|
+
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
|
2125
|
+
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
|
2126
|
+
method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
|
1089
2127
|
method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
|
1090
|
-
method_option :email_notification, :type => :boolean, :aliases => ["
|
1091
|
-
method_option :upload_output, :type => :string, :aliases => ["
|
1092
|
-
method_option :commit, :type => :string, :aliases => ["
|
1093
|
-
method_option :schedule, :type => :string, :aliases => ["
|
1094
|
-
|
2128
|
+
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
|
2129
|
+
method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
|
2130
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
2131
|
+
method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
|
2132
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
|
2133
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
2134
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2135
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2136
|
+
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
|
1095
2137
|
|
1096
2138
|
def run(*cmd)
|
1097
2139
|
sync_before = options["sync_before"]
|
@@ -1103,37 +2145,74 @@ module Cnvrg
|
|
1103
2145
|
upload_output = options["upload_output"]
|
1104
2146
|
local = options["local"]
|
1105
2147
|
schedule = options["schedule"]
|
2148
|
+
image = options["image"]
|
2149
|
+
grid = options["grid"]
|
2150
|
+
data = options["data"]
|
2151
|
+
data_commit = options["data_commit"]
|
2152
|
+
ignore = options["ignore"]
|
2153
|
+
options_hash = Hash[options]
|
2154
|
+
real_options = []
|
2155
|
+
options_hash.each do |o|
|
2156
|
+
real_options << o if (!o[1].eql? "" and !["small", "medium", "large", "gpu", "gpuxl", "gpuxxl"].include? o[0])
|
2157
|
+
end
|
1106
2158
|
if local
|
1107
2159
|
invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
|
1108
|
-
:log => log, :email_notification => email_notification, :upload_output => upload_output,
|
2160
|
+
:log => log, :email_notification => email_notification, :upload_output => upload_output,
|
2161
|
+
:commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
|
1109
2162
|
return
|
1110
2163
|
else
|
1111
|
-
|
2164
|
+
real_options.delete(["local", false])
|
2165
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
2166
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
1112
2167
|
instance_type = get_instance_type(instances)
|
2168
|
+
if !instance_type.nil? and !instance_type.empty?
|
2169
|
+
real_options << ["machine_type", instance_type]
|
2170
|
+
end
|
2171
|
+
exec_options = real_options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
2172
|
+
cmd_to_exec = "#{exec_options} #{cmd.join(" ")}"
|
1113
2173
|
invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
1114
|
-
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit
|
2174
|
+
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
|
2175
|
+
:image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
|
1115
2176
|
return
|
1116
2177
|
end
|
1117
|
-
end
|
1118
2178
|
|
2179
|
+
# if local
|
2180
|
+
#
|
2181
|
+
# else
|
1119
2182
|
|
1120
|
-
desc '', ''
|
1121
|
-
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
|
1122
|
-
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
1123
|
-
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
1124
|
-
method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
|
1125
|
-
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
|
1126
|
-
method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
|
1127
|
-
method_option :commit, :type => :string, :aliases => ["--c", "-c"], :default => ""
|
1128
2183
|
|
2184
|
+
# invoke :exec_remote, [cmd_to_exec.split(" ")], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
2185
|
+
# :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
|
2186
|
+
# :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
|
2187
|
+
# return
|
2188
|
+
# end
|
2189
|
+
end
|
2190
|
+
|
2191
|
+
|
2192
|
+
desc '', '', :hide => true
|
2193
|
+
method_option :sync_before, :type => :boolean, :aliases => ["-sb,--sync_before"], :default => true
|
2194
|
+
method_option :sync_after, :type => :boolean, :aliases => ["-sa,--sync_after"], :default => true
|
2195
|
+
method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
|
2196
|
+
method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
|
2197
|
+
method_option :email_notification, :type => :boolean, :aliases => ["-en,--email_notification"], :default => false
|
2198
|
+
method_option :upload_output, :type => :string, :aliases => ["-uo,--upload_output"], :default => ""
|
2199
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
2200
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
|
2201
|
+
method_option :indocker, :type => :boolean, :aliases => ["--indocker"], :default => false
|
2202
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2203
|
+
method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
|
2204
|
+
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
|
2205
|
+
method_option :remote, :type => :boolean, :aliases => ["--remote"], :default => false
|
2206
|
+
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1129
2207
|
def exec(*cmd)
|
2208
|
+
|
1130
2209
|
log = []
|
1131
2210
|
cpu_average =0
|
1132
2211
|
memory_average = 0
|
1133
2212
|
verify_logged_in(true)
|
1134
2213
|
log_start(__method__, args, options)
|
1135
|
-
working_dir =
|
1136
|
-
|
2214
|
+
working_dir = is_cnvrg_dir
|
2215
|
+
script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
|
1137
2216
|
|
1138
2217
|
|
1139
2218
|
sync_before = options["sync_before"]
|
@@ -1141,8 +2220,14 @@ module Cnvrg
|
|
1141
2220
|
print_log = options["log"]
|
1142
2221
|
title = options["title"]
|
1143
2222
|
commit = options["commit"] || nil
|
2223
|
+
image = options["image"] || nil
|
2224
|
+
indocker = options["indocker"] || false
|
2225
|
+
ignore = options[:ignore] || ""
|
2226
|
+
|
2227
|
+
|
1144
2228
|
email_notification = options["email_notification"]
|
1145
2229
|
upload_output = options["upload_output"]
|
2230
|
+
upload_output = "1m" if upload_output.nil? or upload_output.empty?
|
1146
2231
|
time_to_upload = calc_output_time(upload_output)
|
1147
2232
|
project_home = get_project_home
|
1148
2233
|
@project = Project.new(project_home)
|
@@ -1154,38 +2239,55 @@ module Cnvrg
|
|
1154
2239
|
else
|
1155
2240
|
if sync_before
|
1156
2241
|
# Sync before run
|
1157
|
-
|
2242
|
+
|
2243
|
+
invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
|
1158
2244
|
end
|
1159
2245
|
end
|
2246
|
+
#set image for the project
|
2247
|
+
if !image.nil? and !image.empty?
|
2248
|
+
invoke :set_image, [image]
|
2249
|
+
end
|
2250
|
+
if !indocker
|
2251
|
+
image_proj = is_project_with_docker(working_dir)
|
1160
2252
|
|
1161
2253
|
|
1162
|
-
|
1163
|
-
|
1164
|
-
|
1165
|
-
|
1166
|
-
|
2254
|
+
if image_proj and image_proj.is_docker
|
2255
|
+
container= image_proj.get_container
|
2256
|
+
if !container
|
2257
|
+
say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
|
2258
|
+
exit(1)
|
2259
|
+
end
|
2260
|
+
|
2261
|
+
|
2262
|
+
exec_args = args.flatten.join(" ")
|
2263
|
+
options_hash = Hash[options]
|
2264
|
+
options_hash.except!("image", "indocker")
|
2265
|
+
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
2266
|
+
command_to_run = copy_args.join(" ")
|
2267
|
+
command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
|
2268
|
+
puts container.exec(command, tty: true)
|
2269
|
+
container.stop()
|
2270
|
+
exit(0)
|
1167
2271
|
end
|
1168
|
-
exec_args = args.flatten.join(" ")
|
1169
|
-
exec_options = options.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
1170
|
-
command_to_run = cmd.join(" ")
|
1171
|
-
command = ["/bin/bash", "-lc", "cnvrg exec_docker #{exec_options} #{command_to_run} #{exec_args}"]
|
1172
|
-
puts container.exec(command, tty: true)
|
1173
|
-
container.stop()
|
1174
|
-
exit(0)
|
1175
2272
|
end
|
1176
|
-
|
2273
|
+
remote = options["remote"]
|
2274
|
+
if remote
|
2275
|
+
docker_id = `cat /etc/hostname`
|
2276
|
+
docker_id = docker_id.strip()
|
2277
|
+
end
|
2278
|
+
is_on_gpu = options["gpu"]
|
1177
2279
|
start_commit = @project.last_local_commit
|
1178
2280
|
cmd = cmd.join("\s")
|
1179
2281
|
|
1180
|
-
say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
|
1181
|
-
|
1182
2282
|
@exp = Experiment.new(@project.owner, @project.slug)
|
1183
2283
|
|
1184
2284
|
platform = RUBY_PLATFORM
|
1185
2285
|
machine_name = Socket.gethostname
|
1186
2286
|
begin
|
1187
|
-
machine_activity = @exp.get_machine_activity(
|
1188
|
-
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
|
2287
|
+
machine_activity = @exp.get_machine_activity(working_dir)
|
2288
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
|
2289
|
+
say "Experiment's live results: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
2290
|
+
say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
|
1189
2291
|
unless @exp.slug.nil?
|
1190
2292
|
real = Time.now
|
1191
2293
|
exp_success = true
|
@@ -1197,19 +2299,34 @@ module Cnvrg
|
|
1197
2299
|
begin
|
1198
2300
|
stdout.each do |line|
|
1199
2301
|
cur_time = Time.now
|
1200
|
-
|
1201
|
-
|
1202
|
-
|
1203
|
-
|
2302
|
+
if remote
|
2303
|
+
stats = usage_metrics_in_docker(docker_id)
|
2304
|
+
cpu = stats[0]
|
2305
|
+
memory = stats[1]
|
2306
|
+
else
|
2307
|
+
memory = memory_usage()
|
2308
|
+
cpu = cpu_usage()
|
2309
|
+
end
|
2310
|
+
|
1204
2311
|
memory_total << memory.to_f
|
1205
2312
|
cpu_total << cpu.to_f
|
1206
2313
|
real_time= Time.now-real
|
1207
2314
|
|
1208
|
-
cur_log = {time: cur_time,
|
2315
|
+
cur_log = { time: cur_time,
|
1209
2316
|
message: line,
|
1210
2317
|
type: "stdout",
|
1211
|
-
real: real_time
|
1212
|
-
|
2318
|
+
real: real_time,
|
2319
|
+
cpu: cpu,
|
2320
|
+
memory: memory
|
2321
|
+
}
|
2322
|
+
if is_on_gpu
|
2323
|
+
gpu_stats = gpu_util
|
2324
|
+
gpu_utilization = gpu_stats[0]
|
2325
|
+
gpu_memory_util = gpu_stats[1]
|
2326
|
+
cur_log.merge!(gpu_util:gpu_utilization,gpu_memory_util:gpu_memory_util)
|
2327
|
+
end
|
2328
|
+
|
2329
|
+
if print_log
|
1213
2330
|
puts cur_log
|
1214
2331
|
end
|
1215
2332
|
log << cur_log
|
@@ -1218,8 +2335,8 @@ module Cnvrg
|
|
1218
2335
|
if time_to_upload !=0
|
1219
2336
|
if time_to_upload <= Time.now - start_loop
|
1220
2337
|
#upload current log
|
1221
|
-
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
1222
|
-
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
2338
|
+
# cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
2339
|
+
# memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
1223
2340
|
|
1224
2341
|
@exp.upload_temp_log(log, cpu_average, memory_average)
|
1225
2342
|
log = []
|
@@ -1243,8 +2360,7 @@ module Cnvrg
|
|
1243
2360
|
end
|
1244
2361
|
|
1245
2362
|
rescue Errno::EIO => e
|
1246
|
-
|
1247
|
-
# break
|
2363
|
+
# break
|
1248
2364
|
rescue Errno::ENOENT
|
1249
2365
|
log_end(1, "command #{cmd} isn't valid")
|
1250
2366
|
|
@@ -1256,8 +2372,11 @@ module Cnvrg
|
|
1256
2372
|
exp_success = false
|
1257
2373
|
say "The process exited!", Thor::Shell::Color::RED
|
1258
2374
|
rescue => e
|
1259
|
-
|
1260
|
-
|
2375
|
+
log_end(-1, e.message)
|
2376
|
+
res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
|
2377
|
+
|
2378
|
+
say "Error occurred,aborting", Thor::Shell::Color::RED
|
2379
|
+
exit(0)
|
1261
2380
|
end
|
1262
2381
|
::Process.wait pid
|
1263
2382
|
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
@@ -1268,7 +2387,7 @@ module Cnvrg
|
|
1268
2387
|
end
|
1269
2388
|
if !exp_success
|
1270
2389
|
if !Cnvrg::Helpers.internet_connection?
|
1271
|
-
wait_offline = agree "Seems like you're offline, wait until
|
2390
|
+
wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
|
1272
2391
|
if wait_offline
|
1273
2392
|
say "Waiting until your'e online..", Thor::Shell::Color::BLUE
|
1274
2393
|
while !Cnvrg::Helpers.internet_connection?
|
@@ -1291,23 +2410,21 @@ module Cnvrg
|
|
1291
2410
|
end
|
1292
2411
|
if sync_after
|
1293
2412
|
# Sync after run
|
1294
|
-
if !commit.nil?
|
1295
|
-
invoke :sync, [], :new_branch => true
|
1296
2413
|
|
1297
|
-
|
1298
|
-
|
1299
|
-
|
2414
|
+
|
2415
|
+
download(sync=true, ignore_list=ignore)
|
2416
|
+
upload(link=false, sync=true, direct=false, ignore_list=ignore)
|
2417
|
+
|
1300
2418
|
end
|
1301
2419
|
end_commit = @project.last_local_commit
|
1302
2420
|
|
1303
2421
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
1304
2422
|
check = Helpers.checkmark()
|
1305
|
-
say "#{check} Done. Experiment's
|
2423
|
+
say "#{check} Done. Experiment's results were updated!", Thor::Shell::Color::GREEN
|
1306
2424
|
log_end(0)
|
1307
2425
|
end
|
1308
2426
|
rescue => e
|
1309
|
-
|
1310
|
-
log_end(1, e.message)
|
2427
|
+
log_end(-1, e.message)
|
1311
2428
|
if container
|
1312
2429
|
container.stop()
|
1313
2430
|
end
|
@@ -1334,7 +2451,7 @@ module Cnvrg
|
|
1334
2451
|
end
|
1335
2452
|
end
|
1336
2453
|
|
1337
|
-
desc '', ''
|
2454
|
+
desc '', '', :hide => true
|
1338
2455
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
|
1339
2456
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
1340
2457
|
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
@@ -1373,7 +2490,7 @@ module Cnvrg
|
|
1373
2490
|
machine_name = Socket.gethostname
|
1374
2491
|
begin
|
1375
2492
|
|
1376
|
-
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
|
2493
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
|
1377
2494
|
unless @exp.slug.nil?
|
1378
2495
|
real = Time.now
|
1379
2496
|
exp_success = true
|
@@ -1443,7 +2560,9 @@ module Cnvrg
|
|
1443
2560
|
exp_success = false
|
1444
2561
|
say "The process exited!", Thor::Shell::Color::RED
|
1445
2562
|
rescue => e
|
1446
|
-
log_end(1, e.message)
|
2563
|
+
log_end(-1, e.message)
|
2564
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
2565
|
+
exit(0)
|
1447
2566
|
end
|
1448
2567
|
::Process.wait pid
|
1449
2568
|
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
@@ -1493,7 +2612,7 @@ module Cnvrg
|
|
1493
2612
|
log_end(0)
|
1494
2613
|
end
|
1495
2614
|
rescue => e
|
1496
|
-
log_end(1, e.message)
|
2615
|
+
log_end(-1, e.message)
|
1497
2616
|
say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
|
1498
2617
|
exit(1)
|
1499
2618
|
end
|
@@ -1517,22 +2636,33 @@ module Cnvrg
|
|
1517
2636
|
end
|
1518
2637
|
end
|
1519
2638
|
|
1520
|
-
desc '', ''
|
1521
|
-
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--
|
1522
|
-
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--
|
1523
|
-
method_option :title, :type => :string, :aliases => ["-t", "--
|
1524
|
-
method_option :log, :type => :boolean, :aliases => ["
|
1525
|
-
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--
|
1526
|
-
method_option :upload_output, :type => :string, :aliases => ["
|
1527
|
-
method_option :machine_type, :type => :string, :
|
1528
|
-
method_option :schedule, :type => :string, :aliases => ["
|
1529
|
-
method_option :commit, :type => :string, :aliases => ["
|
2639
|
+
desc '', '', :hide => true
|
2640
|
+
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sync_before"], :default => true
|
2641
|
+
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sync_after"], :default => true
|
2642
|
+
method_option :title, :type => :string, :aliases => ["-t", "--title"], :default => ""
|
2643
|
+
method_option :log, :type => :boolean, :aliases => ["--log"], :default => false
|
2644
|
+
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--email_notification"], :default => false
|
2645
|
+
method_option :upload_output, :type => :string, :aliases => ["-uo", "--upload_output"], :default => ""
|
2646
|
+
method_option :machine_type, :type => :string, :default => ""
|
2647
|
+
method_option :schedule, :type => :string, :aliases => ["-s", "--schedule"], :default => ""
|
2648
|
+
method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
|
2649
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
|
2650
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
2651
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2652
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2653
|
+
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :desc => "ignore following files", :default => ""
|
1530
2654
|
|
1531
2655
|
def exec_remote(*cmd)
|
1532
2656
|
verify_logged_in(true)
|
1533
2657
|
log_start(__method__, args, options)
|
1534
2658
|
working_dir = is_cnvrg_dir
|
2659
|
+
path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
|
2660
|
+
|
1535
2661
|
begin
|
2662
|
+
grid = options["grid"] || nil
|
2663
|
+
data = options["data"] || nil
|
2664
|
+
data_commit = options["data_commit"] || nil
|
2665
|
+
sync_before = options["sync_before"]
|
1536
2666
|
|
1537
2667
|
instance_type = options["machine_type"] || nil
|
1538
2668
|
schedule = options["schedule"] || ""
|
@@ -1554,12 +2684,22 @@ module Cnvrg
|
|
1554
2684
|
new_time = new_time[0, new_time.size-6] #remove timezone
|
1555
2685
|
schedule = "at #{new_time}"
|
1556
2686
|
end
|
1557
|
-
|
2687
|
+
upload_output = options["upload_output"]
|
2688
|
+
time_to_upload = calc_output_time(upload_output)
|
2689
|
+
if time_to_upload==0 or time_to_upload==-1
|
2690
|
+
upload_output_option = "--upload_output=1m"
|
2691
|
+
else
|
2692
|
+
upload_output_option = "--upload_output=#{upload_output}"
|
2693
|
+
end
|
2694
|
+
remote = "--remote=true"
|
2695
|
+
if !instance_type.nil? and instance_type.include? "gpu"
|
2696
|
+
remote= "#{remote} --gpu=true"
|
2697
|
+
end
|
1558
2698
|
|
1559
2699
|
options_hash = Hash[options]
|
1560
|
-
options_hash.except!("schedule", "machine_type")
|
2700
|
+
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "data", "data_commit", "local", "small", "medium", "large", "gpu", "gpuxl", "gpuxxl")
|
1561
2701
|
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
1562
|
-
command = "#{exec_options}
|
2702
|
+
command = "#{exec_options} #{remote} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
1563
2703
|
commit_to_run = options["commit"] || nil
|
1564
2704
|
if !schedule.nil? and !schedule.empty?
|
1565
2705
|
|
@@ -1567,32 +2707,51 @@ module Cnvrg
|
|
1567
2707
|
|
1568
2708
|
end
|
1569
2709
|
project = Project.new(working_dir)
|
2710
|
+
choose_image = options["image"]
|
1570
2711
|
|
1571
|
-
|
2712
|
+
if !choose_image.nil? and !choose_image.empty?
|
2713
|
+
invoke :set_image, [choose_image]
|
2714
|
+
end
|
1572
2715
|
image = is_project_with_docker(working_dir)
|
1573
2716
|
if !image or !image.is_docker
|
1574
|
-
say "Couldn't find image related to project", Thor::Shell::Color::RED
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
image_slug =
|
1579
|
-
else
|
1580
|
-
exit(0)
|
2717
|
+
# say "Couldn't find image related to project", Thor::Shell::Color::RED
|
2718
|
+
|
2719
|
+
image_slug = "cnvrg"
|
2720
|
+
if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
|
2721
|
+
image_slug = "cnvrg_gpu"
|
1581
2722
|
end
|
2723
|
+
# default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
|
2724
|
+
# if default
|
2725
|
+
# image = Images.new(working_dir, default_image_name)
|
2726
|
+
# image_slug = image.image_slug
|
2727
|
+
# else
|
2728
|
+
# exit(0)
|
2729
|
+
# end
|
1582
2730
|
else
|
1583
2731
|
image_slug = image.image_slug
|
1584
2732
|
end
|
1585
2733
|
|
2734
|
+
invoke :sync, [false], [] if sync_before
|
2735
|
+
|
1586
2736
|
|
2737
|
+
if command.include? "'"
|
2738
|
+
oc = command.to_enum(:scan, /'/).map { Regexp.last_match }
|
2739
|
+
pairs = oc.enum_for(:each_slice, 2).to_a
|
2740
|
+
pairs.each_with_index do |p,i|
|
2741
|
+
add=0
|
2742
|
+
if i!=0
|
2743
|
+
add=2*i
|
2744
|
+
end
|
2745
|
+
total_loc = command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add]
|
2746
|
+
command[p[0].offset(0)[0]+add..p[1].offset(0)[0]+add] = "\"#{total_loc}\""
|
2747
|
+
end
|
1587
2748
|
|
1588
|
-
invoke :sync, [], []
|
1589
2749
|
|
2750
|
+
end
|
1590
2751
|
say "Running remote experiment", Thor::Shell::Color::BLUE
|
1591
2752
|
exp = Experiment.new(project.owner, project.slug)
|
1592
|
-
|
1593
|
-
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
2753
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
|
1594
2754
|
if Cnvrg::CLI.is_response_success(res)
|
1595
|
-
|
1596
2755
|
# if res["result"]["machine"] == -1
|
1597
2756
|
# say "There are no available machines", Thor::Shell::Color::BLUE
|
1598
2757
|
# create = yes? "create new machine?", Thor::Shell::Color::YELLOW
|
@@ -1624,7 +2783,13 @@ module Cnvrg
|
|
1624
2783
|
# end
|
1625
2784
|
# else
|
1626
2785
|
check = Helpers.checkmark()
|
1627
|
-
|
2786
|
+
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
2787
|
+
|
2788
|
+
if res["result"]["grid"]
|
2789
|
+
str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
|
2790
|
+
end
|
2791
|
+
|
2792
|
+
say str, Thor::Shell::Color::GREEN
|
1628
2793
|
|
1629
2794
|
exit(0)
|
1630
2795
|
# end
|
@@ -1644,21 +2809,30 @@ module Cnvrg
|
|
1644
2809
|
end
|
1645
2810
|
|
1646
2811
|
desc 'deploy', 'Deploys model to production'
|
1647
|
-
method_option :
|
1648
|
-
method_option :
|
1649
|
-
method_option :
|
2812
|
+
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
2813
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
2814
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
2815
|
+
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
2816
|
+
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
2817
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
2818
|
+
method_option :schedule, :type => :string, :aliases => ["--schedule", "-s"], :default => ""
|
2819
|
+
|
2820
|
+
method_option :commit, :type => :string, :aliases => ["--commit", "-c"], :default => ""
|
2821
|
+
method_option :workers, :type => :string, :aliases => ["--workers", "-w"], :default => ""
|
2822
|
+
method_option :file_as_input, :type => :boolean, :aliases => ["--input", "-i"], :default => false
|
1650
2823
|
|
1651
2824
|
def deploy(file_to_run, function)
|
1652
2825
|
verify_logged_in(true)
|
1653
2826
|
log_start(__method__, args, options)
|
1654
2827
|
working_dir = is_cnvrg_dir
|
1655
2828
|
begin
|
2829
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
2830
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
2831
|
+
instance_type = get_instance_type(instances)
|
1656
2832
|
|
1657
|
-
instance_type = options["machine_type"] || nil
|
1658
2833
|
schedule = options["schedule"] || ""
|
1659
2834
|
|
1660
2835
|
|
1661
|
-
|
1662
2836
|
if !schedule.nil? and !schedule.empty?
|
1663
2837
|
|
1664
2838
|
local_timestamp = get_schedule_date
|
@@ -1667,26 +2841,34 @@ module Cnvrg
|
|
1667
2841
|
project = Project.new(working_dir)
|
1668
2842
|
commit_to_run = options["commit"] || nil
|
1669
2843
|
|
2844
|
+
workers = options["workers"] || nil
|
2845
|
+
begin
|
2846
|
+
num_workers = workers.to_i
|
2847
|
+
rescue
|
2848
|
+
say "Number of workers should be a number between 1 to 10", Thor::Shell::Color::RED
|
2849
|
+
exit(1)
|
2850
|
+
end
|
2851
|
+
file_as_input = options["file_as_input"] || false
|
2852
|
+
|
1670
2853
|
|
1671
2854
|
image = is_project_with_docker(working_dir)
|
1672
2855
|
if !image or !image.is_docker
|
1673
|
-
say "Couldn't find image related to project", Thor::Shell::Color::RED
|
1674
|
-
default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
|
1675
|
-
if default
|
1676
|
-
|
1677
|
-
|
1678
|
-
else
|
1679
|
-
|
1680
|
-
end
|
2856
|
+
# say "Couldn't find image related to project", Thor::Shell::Color::RED
|
2857
|
+
# default = yes? "use cnvrg default image?", Thor::Shell::Color::YELLOW
|
2858
|
+
# if default
|
2859
|
+
image = Images.new(working_dir, "cnvrg")
|
2860
|
+
image_slug = image.image_slug
|
2861
|
+
# else
|
2862
|
+
# exit(0)
|
2863
|
+
# end
|
1681
2864
|
else
|
1682
2865
|
image_slug = image.image_slug
|
1683
2866
|
end
|
1684
2867
|
|
1685
2868
|
|
2869
|
+
invoke :sync, [false], []
|
1686
2870
|
|
1687
|
-
|
1688
|
-
|
1689
|
-
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
2871
|
+
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp, num_workers, file_as_input)
|
1690
2872
|
|
1691
2873
|
if Cnvrg::CLI.is_response_success(res)
|
1692
2874
|
|
@@ -1742,27 +2924,37 @@ module Cnvrg
|
|
1742
2924
|
|
1743
2925
|
method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
|
1744
2926
|
method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
1745
|
-
method_option :local, :type => :boolean, :aliases => ["
|
1746
|
-
method_option :small, :type => :boolean, :aliases => ["
|
1747
|
-
method_option :medium, :type => :boolean, :aliases => ["--
|
1748
|
-
method_option :large, :type => :boolean, :aliases => ["
|
2927
|
+
method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
|
2928
|
+
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
2929
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
2930
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
1749
2931
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1750
|
-
method_option :gpuxl, :type => :boolean, :aliases => ["--
|
1751
|
-
|
2932
|
+
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
2933
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
2934
|
+
method_option :image, :type => :string, :aliases => ["-i", "--image"], :default => ""
|
2935
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2936
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2937
|
+
|
2938
|
+
|
2939
|
+
desc 'notebook', 'starts a notebook session remotely or locally'
|
1752
2940
|
|
1753
|
-
def notebook
|
2941
|
+
def notebook
|
1754
2942
|
local = options["local"]
|
1755
2943
|
notebook_dir = options["notebook_dir"]
|
1756
2944
|
kernel = options["kernel"]
|
1757
|
-
|
2945
|
+
image = options["image"]
|
2946
|
+
data = options["data"]
|
2947
|
+
data_commit = options["data_commit"]
|
1758
2948
|
if local
|
1759
|
-
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel
|
2949
|
+
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
|
1760
2950
|
return
|
1761
2951
|
else
|
1762
|
-
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
2952
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
2953
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
1763
2954
|
instance_type = get_instance_type(instances)
|
1764
2955
|
|
1765
|
-
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type
|
2956
|
+
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image,
|
2957
|
+
:data => data, :data_commit => data_commit
|
1766
2958
|
return
|
1767
2959
|
|
1768
2960
|
end
|
@@ -1771,9 +2963,13 @@ module Cnvrg
|
|
1771
2963
|
end
|
1772
2964
|
|
1773
2965
|
desc 'remote_notebook', 'run notebook server on remote server'
|
1774
|
-
method_option :notebook_dir, :type => :string, :aliases => ["-n"
|
1775
|
-
method_option :machine_type, :type => :string, :
|
1776
|
-
method_option :kernel, :type => :string, :aliases => ["--
|
2966
|
+
method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
2967
|
+
method_option :machine_type, :type => :string, :default => ""
|
2968
|
+
method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
|
2969
|
+
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
2970
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2971
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2972
|
+
|
1777
2973
|
|
1778
2974
|
def remote_notebook()
|
1779
2975
|
verify_logged_in(true)
|
@@ -1784,23 +2980,35 @@ module Cnvrg
|
|
1784
2980
|
notebook_dir = options["notebook_dir"]
|
1785
2981
|
instance_type = options["machine_type"] || nil
|
1786
2982
|
kernel = options["kernel"] || nil
|
2983
|
+
data = options["data"]
|
2984
|
+
data_commit = options["data_commit"]
|
1787
2985
|
|
1788
2986
|
|
1789
2987
|
begin
|
2988
|
+
choose_image = options["image"]
|
1790
2989
|
|
2990
|
+
if !choose_image.nil? and !choose_image.empty?
|
2991
|
+
invoke :set_image, [choose_image]
|
2992
|
+
end
|
1791
2993
|
@image = is_project_with_docker(working_dir)
|
1792
|
-
if !@image.is_docker
|
1793
|
-
say "Couldn't find image related to project", Thor::Shell::Color::RED
|
1794
|
-
|
2994
|
+
if !@image or !@image.is_docker
|
2995
|
+
# say "Couldn't find image related to project", Thor::Shell::Color::RED
|
2996
|
+
default_image_name = "cnvrg"
|
2997
|
+
if instance_type.eql? "gpu" or instance_type.eql? "gpuxl"
|
2998
|
+
default_image_name = "cnvrg-gpu"
|
2999
|
+
end
|
3000
|
+
# default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
|
3001
|
+
# if default
|
3002
|
+
@image = Images.new(working_dir, default_image_name)
|
3003
|
+
# else
|
3004
|
+
# exit(0)
|
3005
|
+
# end
|
1795
3006
|
end
|
1796
3007
|
|
1797
|
-
|
1798
|
-
|
1799
|
-
download()
|
1800
|
-
upload()
|
3008
|
+
invoke :sync, [false], []
|
1801
3009
|
|
1802
3010
|
|
1803
|
-
res = @image.remote_notebook(notebook_dir, instance_type, kernel)
|
3011
|
+
res = @image.remote_notebook(notebook_dir, instance_type, kernel, data, data_commit)
|
1804
3012
|
if Cnvrg::CLI.is_response_success(res)
|
1805
3013
|
if res["result"]["machine"] == -1
|
1806
3014
|
say "There are no available machines", Thor::Shell::Color::BLUE
|
@@ -1839,7 +3047,7 @@ module Cnvrg
|
|
1839
3047
|
note_url = res["result"]["notebook_url"]
|
1840
3048
|
@image.set_note_url(note_url)
|
1841
3049
|
check = Helpers.checkmark()
|
1842
|
-
say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@
|
3050
|
+
say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
|
1843
3051
|
# Launchy.open(url)
|
1844
3052
|
|
1845
3053
|
exit(0)
|
@@ -1907,9 +3115,8 @@ module Cnvrg
|
|
1907
3115
|
|
1908
3116
|
end
|
1909
3117
|
rescue => e
|
1910
|
-
|
1911
|
-
|
1912
|
-
say "Error occurd, Aborting"
|
3118
|
+
log_end(-1, e.message)
|
3119
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
1913
3120
|
if container
|
1914
3121
|
container.stop()
|
1915
3122
|
end
|
@@ -1970,9 +3177,8 @@ module Cnvrg
|
|
1970
3177
|
end
|
1971
3178
|
end
|
1972
3179
|
rescue => e
|
1973
|
-
|
1974
|
-
|
1975
|
-
say "Error occurd, Aborting"
|
3180
|
+
log_end(-1, e.message)
|
3181
|
+
say "Error occurred, aborting"
|
1976
3182
|
if container
|
1977
3183
|
container.stop()
|
1978
3184
|
end
|
@@ -1989,6 +3195,7 @@ module Cnvrg
|
|
1989
3195
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
|
1990
3196
|
method_option :kernel, :type => :string, :aliases => ["-k", "--k"], :default => "", :desc => "default kernel"
|
1991
3197
|
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
3198
|
+
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1992
3199
|
|
1993
3200
|
def run_notebook
|
1994
3201
|
|
@@ -2002,7 +3209,7 @@ module Cnvrg
|
|
2002
3209
|
notebook_dir = options["notebook_dir"]
|
2003
3210
|
remote = options["remote"] || false
|
2004
3211
|
kernel = options["kernel"] || ""
|
2005
|
-
|
3212
|
+
notebooks_pid = nil
|
2006
3213
|
|
2007
3214
|
if notebook_dir.empty?
|
2008
3215
|
notebook_dir = project_dir
|
@@ -2010,107 +3217,148 @@ module Cnvrg
|
|
2010
3217
|
|
2011
3218
|
notebook_dir = project_dir+ notebook_dir
|
2012
3219
|
end
|
2013
|
-
|
2014
|
-
image = is_project_with_docker(Dir.pwd)
|
2015
|
-
if image and image.is_docker and !remote
|
2016
|
-
container= image.get_container
|
2017
|
-
if !container
|
3220
|
+
choose_image = options["image"]
|
2018
3221
|
|
2019
|
-
|
3222
|
+
if !choose_image.nil? and !choose_image.empty?
|
3223
|
+
invoke :set_image, [choose_image]
|
3224
|
+
end
|
3225
|
+
|
3226
|
+
image = is_project_with_docker(Dir.pwd)
|
3227
|
+
if !image
|
3228
|
+
jupyter_installed = `which jupyter`
|
3229
|
+
if !$?.success?
|
3230
|
+
say "Could not find jupyter, Is it installed?", Thor::Shell::Color::RED
|
2020
3231
|
exit(1)
|
2021
3232
|
end
|
2022
|
-
else
|
2023
|
-
say "Project is not configured with any image", Thor::Shell::Color::RED
|
2024
|
-
exit(1)
|
2025
3233
|
|
2026
|
-
end
|
2027
|
-
if options["verbose"]
|
2028
|
-
say "Syncing project before running", Thor::Shell::Color::BLUE
|
2029
|
-
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
2030
|
-
end
|
2031
|
-
@project = Project.new(project_dir)
|
2032
3234
|
|
2033
|
-
|
3235
|
+
cmd = "jupyter-notebook --port=8888"
|
3236
|
+
PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
|
3237
|
+
begin
|
3238
|
+
notebooks_pid = pid
|
3239
|
+
stdout.each do |line|
|
3240
|
+
puts line
|
3241
|
+
|
3242
|
+
end
|
3243
|
+
|
3244
|
+
rescue Errno::EIO => e
|
3245
|
+
# break
|
3246
|
+
rescue Errno::ENOENT
|
3247
|
+
log_end(1, "command #{cmd} isn't valid")
|
3248
|
+
|
3249
|
+
|
3250
|
+
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
3251
|
+
rescue PTY::ChildExited
|
3252
|
+
log_end(1, "proccess exited")
|
3253
|
+
say "The process exited!", Thor::Shell::Color::RED
|
3254
|
+
rescue => e
|
3255
|
+
log_end(-1, e.message)
|
3256
|
+
say "Error occurred,aborting", Thor::Shell::Color::RED
|
3257
|
+
exit(0)
|
3258
|
+
|
3259
|
+
end
|
3260
|
+
|
2034
3261
|
|
2035
|
-
if (note_slug=image.note_slug)
|
2036
|
-
say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
|
2037
|
-
new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
|
2038
|
-
if !new
|
2039
|
-
exit(0)
|
2040
3262
|
end
|
2041
3263
|
|
2042
3264
|
end
|
2043
|
-
invoke :sync, [], :verbose => options["verbose"]
|
2044
|
-
say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
|
2045
|
-
#replace url
|
2046
|
-
base_url = get_base_url()
|
2047
3265
|
|
2048
|
-
|
2049
|
-
|
2050
|
-
|
2051
|
-
|
2052
|
-
|
3266
|
+
if image and image.is_docker and !remote
|
3267
|
+
container= image.get_container
|
3268
|
+
if !container
|
3269
|
+
say "Couldn't start docker container", Thor::Shell::Color::RED
|
3270
|
+
exit(1)
|
2053
3271
|
|
2054
|
-
|
2055
|
-
port = image.container_port()
|
3272
|
+
end
|
2056
3273
|
|
2057
|
-
|
2058
|
-
|
2059
|
-
|
2060
|
-
|
2061
|
-
|
2062
|
-
|
2063
|
-
|
3274
|
+
if options["verbose"]
|
3275
|
+
say "Syncing project before running", Thor::Shell::Color::BLUE
|
3276
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
3277
|
+
end
|
3278
|
+
@project = Project.new(project_dir)
|
3279
|
+
|
3280
|
+
start_commit = @project.last_local_commit
|
3281
|
+
|
3282
|
+
if (note_slug=image.note_slug)
|
3283
|
+
say "There is a running notebook session in: https://cnvrg.io/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_slug}", Thor::Shell::Color::BLUE
|
3284
|
+
new = yes? "Create a new session?", Thor::Shell::Color::YELLOW
|
3285
|
+
if !new
|
3286
|
+
exit(0)
|
3287
|
+
end
|
3288
|
+
|
3289
|
+
end
|
3290
|
+
invoke :sync, [false], :verbose => options["verbose"]
|
3291
|
+
say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
|
3292
|
+
#replace url
|
3293
|
+
base_url = get_base_url()
|
3294
|
+
|
3295
|
+
local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
|
3296
|
+
command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
3297
|
+
container.exec(command, tty: true)
|
3298
|
+
container.stop()
|
3299
|
+
container.start()
|
3300
|
+
sleep(7)
|
3301
|
+
@note = Experiment.new(@project.owner, @project.slug)
|
3302
|
+
port = image.container_port()
|
3303
|
+
|
3304
|
+
command = ["/bin/bash", "-lc", "jupyter notebook list"]
|
3305
|
+
list = container.exec(command, tty: true)[0]
|
3306
|
+
if list.empty? or list.nil?
|
3307
|
+
say "Couldn't start notebook server", Thor::Shell::Color::RED
|
3308
|
+
log_end(1, "can't start notebook server")
|
3309
|
+
exit(1)
|
3310
|
+
end
|
2064
3311
|
|
2065
|
-
|
2066
|
-
|
3312
|
+
result = ""
|
3313
|
+
list.each do |r|
|
3314
|
+
if r.include? "http"
|
3315
|
+
result = r
|
3316
|
+
end
|
3317
|
+
end
|
3318
|
+
token = result.to_s.split("::")[0].to_s.match(/(token=)(.+)\s/)[2]
|
2067
3319
|
|
2068
|
-
|
3320
|
+
# machine_activity = @note.get_machine_activity(project_dir)
|
2069
3321
|
|
2070
3322
|
|
2071
|
-
|
2072
|
-
|
2073
|
-
|
3323
|
+
slug = @note.start_notebook_session(kernel, start_commit, token, port, false, notebook_dir)
|
3324
|
+
image.set_note_url(slug)
|
3325
|
+
note_url = "http://localhost:#{port}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local/?token=#{token}"
|
2074
3326
|
|
2075
3327
|
|
2076
|
-
|
2077
|
-
|
3328
|
+
if !note_url.empty?
|
3329
|
+
check = Helpers.checkmark()
|
2078
3330
|
|
2079
|
-
|
2080
|
-
|
2081
|
-
|
2082
|
-
|
2083
|
-
|
2084
|
-
|
3331
|
+
say "#{check} Notebook server started successfully", Thor::Shell::Color::GREEN
|
3332
|
+
Launchy.open(note_url)
|
3333
|
+
else
|
3334
|
+
say "Couldn't start notebook server", Thor::Shell::Color::RED
|
3335
|
+
log_end(1, "can't start notebook server")
|
3336
|
+
exit(1)
|
3337
|
+
end
|
2085
3338
|
end
|
2086
3339
|
|
2087
|
-
|
2088
|
-
# logs = `#{jup} --no-browser --ip=0.0.0.0 --notebook-dir=#{notebook_dir}`
|
2089
|
-
# url = URI.extract(logs).reject { |x| x if !x.include? "http" }.uniq![0]
|
2090
|
-
# if !url.empty?
|
2091
|
-
# check = Helpers.checkmark()
|
2092
|
-
#
|
2093
|
-
# say "#{check} Notebook server started successfully, view notebook in url: #{url}", Thor::Shell::Color::GREEN
|
2094
|
-
# log_end(0)
|
2095
|
-
# else
|
2096
|
-
# say "Couldn't start notebook server", Thor::Shell::Color::RED
|
2097
|
-
# log_end(1, "can't start notebook server")
|
2098
|
-
# exit(1)
|
2099
|
-
# end
|
3340
|
+
|
2100
3341
|
rescue => e
|
2101
|
-
log_end(-1)
|
2102
|
-
puts e
|
3342
|
+
log_end(-1, e.message)
|
2103
3343
|
say "Error occurd, aborting", Thor::Shell::Color::RED
|
2104
3344
|
if container
|
2105
3345
|
container.stop()
|
2106
3346
|
end
|
2107
3347
|
rescue SignalException
|
2108
|
-
|
2109
|
-
|
2110
|
-
|
3348
|
+
if !notebooks_pid.nil?
|
3349
|
+
::Process.kill(0, notebooks_pid)
|
3350
|
+
say "#{check} Notebook has stopped successfully", Thor::Shell::Color::GREEN
|
3351
|
+
|
3352
|
+
|
3353
|
+
invoke :sync, [false], []
|
3354
|
+
else
|
3355
|
+
log_end(-1)
|
3356
|
+
|
3357
|
+
if container
|
3358
|
+
container.stop()
|
3359
|
+
end
|
2111
3360
|
end
|
2112
|
-
|
2113
|
-
exit(1)
|
3361
|
+
|
2114
3362
|
end
|
2115
3363
|
|
2116
3364
|
|
@@ -2131,7 +3379,7 @@ module Cnvrg
|
|
2131
3379
|
|
2132
3380
|
|
2133
3381
|
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
|
2134
|
-
invoke :sync, [], :verbose => options["verbose"]
|
3382
|
+
invoke :sync, [false], :verbose => options["verbose"]
|
2135
3383
|
|
2136
3384
|
say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
|
2137
3385
|
|
@@ -2176,8 +3424,7 @@ module Cnvrg
|
|
2176
3424
|
exit(0)
|
2177
3425
|
end
|
2178
3426
|
rescue => e
|
2179
|
-
|
2180
|
-
log_end(-1)
|
3427
|
+
log_end(-1, e.message)
|
2181
3428
|
say "Error occurd, aborting"
|
2182
3429
|
if container
|
2183
3430
|
container.stop()
|
@@ -2225,18 +3472,170 @@ module Cnvrg
|
|
2225
3472
|
say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
|
2226
3473
|
container.stop()
|
2227
3474
|
|
2228
|
-
log_end(0)
|
2229
|
-
rescue => e
|
2230
|
-
log_end(-1)
|
3475
|
+
log_end(0)
|
3476
|
+
rescue => e
|
3477
|
+
log_end(-1, e.message)
|
3478
|
+
say "Error occurd, aborting"
|
3479
|
+
if container
|
3480
|
+
container.stop()
|
3481
|
+
end
|
3482
|
+
rescue SignalException
|
3483
|
+
log_End(-1)
|
3484
|
+
if container
|
3485
|
+
container.stop()
|
3486
|
+
end
|
3487
|
+
say "\nAborting"
|
3488
|
+
exit(1)
|
3489
|
+
end
|
3490
|
+
|
3491
|
+
end
|
3492
|
+
|
3493
|
+
desc 'install_libraries', 'Install libraries'
|
3494
|
+
method_option :requirement, :type => :string, :aliases => ["-r", "--r"], :default => "", :desc => "Install from the given requirements file"
|
3495
|
+
|
3496
|
+
def install_python_libraries(*lib)
|
3497
|
+
begin
|
3498
|
+
verify_logged_in(false)
|
3499
|
+
log_start(__method__, args, options)
|
3500
|
+
verify_software_installed("docker")
|
3501
|
+
image = is_project_with_docker(Dir.pwd)
|
3502
|
+
if image and image.is_docker
|
3503
|
+
container = image.get_container
|
3504
|
+
if !container
|
3505
|
+
|
3506
|
+
say "Couldn't create container with image #{image.image_name}:#{image.image_tag}", Thor::Shell::Color::RED
|
3507
|
+
exit(1)
|
3508
|
+
end
|
3509
|
+
else
|
3510
|
+
say "Project is not configured with any image", Thor::Shell::Color::RED
|
3511
|
+
exit(1)
|
3512
|
+
|
3513
|
+
end
|
3514
|
+
req_file = options["requirement"] || nil
|
3515
|
+
if lib.nil? and not req_file.nil?
|
3516
|
+
if not File.exist? req_file
|
3517
|
+
say "Couldn't find #{req_file}", Thor::Shell::Color::RED
|
3518
|
+
exit(1)
|
3519
|
+
|
3520
|
+
end
|
3521
|
+
command_to_run = "pip install -r #{req_file}"
|
3522
|
+
|
3523
|
+
else
|
3524
|
+
command_to_run = lib.join(" ")
|
3525
|
+
|
3526
|
+
end
|
3527
|
+
say "Running #{command_to_run} in container", Thor::Shell::Color::BLUE
|
3528
|
+
command = ["/bin/bash", "-lc", "#{command_to_run}"]
|
3529
|
+
res = container.exec(command, tty: false)
|
3530
|
+
say res[0].join("\n")
|
3531
|
+
checks = Helpers.checkmark()
|
3532
|
+
say "Updating image", Thor::Shell::Color::BLUE
|
3533
|
+
|
3534
|
+
image.create_custom_image("")
|
3535
|
+
say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
|
3536
|
+
container.stop()
|
3537
|
+
|
3538
|
+
log_end(0)
|
3539
|
+
rescue => e
|
3540
|
+
log_end(-1, e.message)
|
3541
|
+
say "Error occurd, aborting"
|
3542
|
+
if container
|
3543
|
+
container.stop()
|
3544
|
+
end
|
3545
|
+
rescue SignalException
|
3546
|
+
if container
|
3547
|
+
container.stop()
|
3548
|
+
end
|
3549
|
+
say "\nAborting"
|
3550
|
+
exit(1)
|
3551
|
+
end
|
3552
|
+
|
3553
|
+
end
|
3554
|
+
|
3555
|
+
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
3556
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
3557
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
3558
|
+
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
3559
|
+
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
3560
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
3561
|
+
method_option :image, :type => :string, :aliases => ["-i","--image"], :default => ""
|
3562
|
+
method_option :public, :type => :boolean, :aliases => ["-p","--public"], :default => false
|
3563
|
+
method_option :base, :type => :boolean, :aliases => ["-b","--base"], :default => false
|
3564
|
+
method_option :python3, :type => :boolean, :aliases => ["--python3"], :default => false
|
3565
|
+
|
3566
|
+
desc 'create_custom_image', 'run commands inside containers', :hide=>true
|
3567
|
+
|
3568
|
+
def create_custom_image(image_name)
|
3569
|
+
begin
|
3570
|
+
verify_logged_in(false)
|
3571
|
+
log_start(__method__, args, options)
|
3572
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
3573
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
3574
|
+
instance_type = get_instance_type(instances)
|
3575
|
+
image_extend = options["image"]
|
3576
|
+
public = options["public"]
|
3577
|
+
base = options["base"]
|
3578
|
+
python3 = options["python3"]
|
3579
|
+
owner = CLI.get_owner
|
3580
|
+
checks = Helpers.checkmark()
|
3581
|
+
|
3582
|
+
say "Creating machine for custom image, this may take a few moments...", Thor::Shell::Color::BLUE
|
3583
|
+
|
3584
|
+
resp = Images.create_new_custom_image(instance_type,owner,image_name,public,base,image_extend,python3)
|
3585
|
+
if Cnvrg::CLI.is_response_success(resp,false)
|
3586
|
+
image_slug = resp["result"]["slug"]
|
3587
|
+
container = resp["result"]["machine_c"]
|
3588
|
+
say "#{checks} Created image and machine successfully", Thor::Shell::Color::GREEN
|
3589
|
+
say "Connecting to machine", Thor::Shell::Color::BLUE
|
3590
|
+
ssh = Ssh.new(resp)
|
3591
|
+
if !ssh.is_ssh
|
3592
|
+
say "Couldn't connect to machine,aborting", Thor::Shell::Color::RED
|
3593
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3594
|
+
log_end(-1,"Couldn't connect to machine,aborting")
|
3595
|
+
end
|
3596
|
+
say "run command until ctrl + c initiates", Thor::Shell::Color::BLUE
|
3597
|
+
begin
|
3598
|
+
|
3599
|
+
while true
|
3600
|
+
command = ask("$>")
|
3601
|
+
puts ssh.exec_command(command)
|
3602
|
+
end
|
3603
|
+
|
3604
|
+
rescue SignalException
|
3605
|
+
say "Commiting Image..", Thor::Shell::Color::BLUE
|
3606
|
+
|
3607
|
+
end
|
3608
|
+
resp = Images.commit_custom_image(owner,image_slug)
|
3609
|
+
if Cnvrg::CLI.is_response_success(resp,false)
|
3610
|
+
say "#{checks} Image commited successfuly, email will be sent when image is ready", Thor::Shell::Color::GREEN
|
3611
|
+
end
|
3612
|
+
if ssh
|
3613
|
+
ssh.close_ssh()
|
3614
|
+
end
|
3615
|
+
|
3616
|
+
|
3617
|
+
|
3618
|
+
|
3619
|
+
|
3620
|
+
|
3621
|
+
|
3622
|
+
end
|
3623
|
+
rescue =>e
|
2231
3624
|
puts e
|
2232
|
-
|
2233
|
-
|
2234
|
-
|
3625
|
+
if image_slug
|
3626
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3627
|
+
end
|
3628
|
+
if ssh
|
3629
|
+
ssh.close_ssh()
|
2235
3630
|
end
|
3631
|
+
|
3632
|
+
|
2236
3633
|
rescue SignalException
|
2237
|
-
|
2238
|
-
|
2239
|
-
|
3634
|
+
if image_slug
|
3635
|
+
Images.revoke_custom_new_image(owner, image_slug)
|
3636
|
+
end
|
3637
|
+
if ssh
|
3638
|
+
ssh.close_ssh
|
2240
3639
|
end
|
2241
3640
|
say "\nAborting"
|
2242
3641
|
exit(1)
|
@@ -2244,15 +3643,26 @@ module Cnvrg
|
|
2244
3643
|
|
2245
3644
|
end
|
2246
3645
|
|
2247
|
-
desc 'install_libraries', 'Install libraries'
|
2248
|
-
method_option :requirement, :type => :string, :aliases => ["-r", "--r"], :default => "", :desc => "Install from the given requirements file"
|
2249
3646
|
|
2250
|
-
|
3647
|
+
desc 'build', 'run commands inside containers'
|
3648
|
+
method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
|
3649
|
+
|
3650
|
+
def build(*cmd)
|
2251
3651
|
begin
|
2252
3652
|
verify_logged_in(false)
|
2253
3653
|
log_start(__method__, args, options)
|
2254
3654
|
verify_software_installed("docker")
|
2255
|
-
|
3655
|
+
working_dir = is_cnvrg_dir
|
3656
|
+
install_file = options["install"] || nil
|
3657
|
+
if !install_file.nil?
|
3658
|
+
commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
|
3659
|
+
|
3660
|
+
else
|
3661
|
+
commands = [cmd.join(" ")]
|
3662
|
+
end
|
3663
|
+
|
3664
|
+
|
3665
|
+
image = is_project_with_docker(working_dir)
|
2256
3666
|
if image and image.is_docker
|
2257
3667
|
container = image.get_container
|
2258
3668
|
if !container
|
@@ -2265,35 +3675,38 @@ module Cnvrg
|
|
2265
3675
|
exit(1)
|
2266
3676
|
|
2267
3677
|
end
|
2268
|
-
|
2269
|
-
|
2270
|
-
|
2271
|
-
|
2272
|
-
|
2273
|
-
|
3678
|
+
commands.each do |c|
|
3679
|
+
if c.include? "pip"
|
3680
|
+
c.sub("pip", "/opt/ds/bin/pip")
|
3681
|
+
end
|
3682
|
+
if c.include? "pip3"
|
3683
|
+
c.sub("pip3", "/opt/ds3/bin/pip3")
|
2274
3684
|
end
|
2275
|
-
command_to_run = "pip install -r #{req_file}"
|
2276
3685
|
|
2277
|
-
|
2278
|
-
|
3686
|
+
say "Running #{c}", Thor::Shell::Color::BLUE
|
3687
|
+
command = ["/bin/bash", "-lc", "#{c}"]
|
3688
|
+
res = container.exec(command, tty: false)
|
3689
|
+
if res[2] != 0
|
3690
|
+
say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
|
3691
|
+
container.stop()
|
3692
|
+
log_end(0)
|
2279
3693
|
|
3694
|
+
exit(1)
|
3695
|
+
end
|
3696
|
+
say res[0].join("\n")
|
3697
|
+
image.store_image_build_commands(working_dir, c)
|
2280
3698
|
end
|
2281
|
-
|
2282
|
-
command = ["/bin/bash", "-lc", "#{command_to_run}"]
|
2283
|
-
res = container.exec(command, tty: false)
|
2284
|
-
say res[0].join("\n")
|
3699
|
+
|
2285
3700
|
checks = Helpers.checkmark()
|
2286
3701
|
say "Updating image", Thor::Shell::Color::BLUE
|
2287
|
-
|
2288
|
-
image.create_custom_image("")
|
2289
|
-
say "#{checks} Done, installing libraries completed", Thor::Shell::Color::GREEN
|
3702
|
+
# image.create_custom_image("",working_dir)
|
2290
3703
|
container.stop()
|
3704
|
+
say "#{checks} Done", Thor::Shell::Color::GREEN
|
2291
3705
|
|
2292
3706
|
log_end(0)
|
2293
3707
|
rescue => e
|
2294
|
-
log_end(-1)
|
2295
|
-
|
2296
|
-
say "Error occurd, aborting"
|
3708
|
+
log_end(-1, e.message)
|
3709
|
+
say "Error occurd, aborting", Thor::Shell::Color::RED
|
2297
3710
|
if container
|
2298
3711
|
container.stop()
|
2299
3712
|
end
|
@@ -2341,8 +3754,9 @@ module Cnvrg
|
|
2341
3754
|
log_end(0)
|
2342
3755
|
return new_image.id
|
2343
3756
|
rescue => e
|
2344
|
-
|
2345
|
-
|
3757
|
+
log_end(-1, e.message)
|
3758
|
+
say "\nError occurred,aborting"
|
3759
|
+
exit(1)
|
2346
3760
|
rescue SignalException
|
2347
3761
|
log_end(-1)
|
2348
3762
|
say "\nAborting"
|
@@ -2385,15 +3799,13 @@ module Cnvrg
|
|
2385
3799
|
|
2386
3800
|
end
|
2387
3801
|
|
2388
|
-
desc '
|
2389
|
-
method_option :name, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "new image name"
|
3802
|
+
desc 'push', 'push image to cnvrg repository'
|
2390
3803
|
|
2391
|
-
def
|
3804
|
+
def push(*name)
|
2392
3805
|
verify_logged_in(true)
|
2393
3806
|
log_start(__method__, args, options)
|
2394
3807
|
working_dir = is_cnvrg_dir()
|
2395
|
-
|
2396
|
-
if !image_name.empty? and image_name == "cnvrg"
|
3808
|
+
if !name.empty? and name == "cnvrg"
|
2397
3809
|
say "can't create image with the name cnvrg", Thor::Shell::Color::RED
|
2398
3810
|
exit(1)
|
2399
3811
|
end
|
@@ -2403,12 +3815,26 @@ module Cnvrg
|
|
2403
3815
|
say "Couldn't find image related to project", Thor::Shell::Color::RED
|
2404
3816
|
exit(0)
|
2405
3817
|
end
|
3818
|
+
if !name.nil? and !name.empty?
|
3819
|
+
if name.include? " "
|
3820
|
+
name.gsub!(" ", "_")
|
3821
|
+
end
|
3822
|
+
end
|
3823
|
+
stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
|
3824
|
+
if stored_commands.nil? or stored_commands.empty?
|
3825
|
+
say "Nothing to push", Thor::Shell::Color::BLUE
|
3826
|
+
exit(0)
|
3827
|
+
end
|
2406
3828
|
|
2407
|
-
image
|
2408
|
-
|
3829
|
+
say "Pushing new image", Thor::Shell::Color::BLUE
|
3830
|
+
if image.create_custom_image(name, working_dir, stored_commands)
|
2409
3831
|
|
3832
|
+
say "Image was updated successfully", Thor::Shell::Color::GREEN
|
3833
|
+
end
|
2410
3834
|
rescue => e
|
2411
|
-
|
3835
|
+
log_end(-1, e.message)
|
3836
|
+
say "error occurred, aborting"
|
3837
|
+
|
2412
3838
|
end
|
2413
3839
|
end
|
2414
3840
|
|
@@ -2470,8 +3896,8 @@ module Cnvrg
|
|
2470
3896
|
exit(1)
|
2471
3897
|
end
|
2472
3898
|
rescue => e
|
2473
|
-
|
2474
|
-
|
3899
|
+
log_end(-1, e.message)
|
3900
|
+
say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
|
2475
3901
|
rescue SignalException
|
2476
3902
|
log_end(-1)
|
2477
3903
|
|
@@ -2480,7 +3906,7 @@ module Cnvrg
|
|
2480
3906
|
end
|
2481
3907
|
end
|
2482
3908
|
|
2483
|
-
desc '', ''
|
3909
|
+
desc '', '', :hide => true
|
2484
3910
|
|
2485
3911
|
def upload_log()
|
2486
3912
|
log_path = '/home/ds/app/uwsgi.log'
|
@@ -2489,28 +3915,32 @@ module Cnvrg
|
|
2489
3915
|
|
2490
3916
|
end
|
2491
3917
|
|
2492
|
-
desc '', ''
|
3918
|
+
desc '', '', :hide => true
|
2493
3919
|
|
2494
3920
|
def exec_container(container_id, *cmd)
|
2495
3921
|
container = Docker::Container.get(container_id)
|
2496
3922
|
container.start()
|
2497
3923
|
cnvrg_command = cmd.join(" ")
|
2498
3924
|
command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
|
2499
|
-
res = container.exec(command, tty: true,wait:5400)[0]
|
3925
|
+
res = container.exec(command, tty: true, wait: 5400)[0]
|
2500
3926
|
say res
|
2501
|
-
|
2502
3927
|
end
|
2503
3928
|
|
2504
|
-
desc '', ''
|
3929
|
+
desc '', '', :hide => true
|
2505
3930
|
|
2506
3931
|
def port_container(container_id)
|
2507
3932
|
container = Docker::Container.get(container_id)
|
2508
3933
|
say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
|
3934
|
+
end
|
2509
3935
|
|
3936
|
+
desc '', '', :hide => true
|
2510
3937
|
|
3938
|
+
def tensor_port_container(container_id)
|
3939
|
+
container = Docker::Container.get(container_id)
|
3940
|
+
say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
|
2511
3941
|
end
|
2512
3942
|
|
2513
|
-
desc '', ''
|
3943
|
+
desc '', '', :hide => true
|
2514
3944
|
|
2515
3945
|
def stop_container(container_id)
|
2516
3946
|
container = Docker::Container.get(container_id)
|
@@ -2519,13 +3949,13 @@ module Cnvrg
|
|
2519
3949
|
|
2520
3950
|
end
|
2521
3951
|
|
2522
|
-
desc '', ''
|
2523
|
-
method_option :login, :type => :string, :aliases => ["-l"
|
2524
|
-
method_option :app_dir, :type => :string, :aliases => ["-d"
|
2525
|
-
method_option :cmd, :type => :string, :aliases => ["-c"
|
3952
|
+
desc '', '', :hide => true
|
3953
|
+
method_option :login, :type => :string, :aliases => ["-l"], :default => ""
|
3954
|
+
method_option :app_dir, :type => :string, :aliases => ["-d"], :default => "/home/ds/notebooks"
|
3955
|
+
method_option :cmd, :type => :string, :aliases => ["-c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
2526
3956
|
|
2527
3957
|
|
2528
|
-
def config_remote(image_name, port=7654)
|
3958
|
+
def config_remote(image_name, port=7654, tensport=6006)
|
2529
3959
|
local_images = Docker::Image.all
|
2530
3960
|
|
2531
3961
|
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
|
@@ -2538,20 +3968,25 @@ module Cnvrg
|
|
2538
3968
|
login_content = options["login"]
|
2539
3969
|
app_dir = options["app_dir"]
|
2540
3970
|
cmd= options["cmd"]
|
3971
|
+
volume_from = options["volume"]
|
2541
3972
|
|
2542
3973
|
image_settings = {
|
2543
3974
|
'Image' => "#{image_name}:latest",
|
2544
|
-
|
3975
|
+
|
2545
3976
|
'Cmd' => cmd,
|
2546
3977
|
'WorkingDir' => app_dir,
|
2547
3978
|
'ExposedPorts' => {
|
2548
3979
|
'8888/tcp' => {},
|
2549
3980
|
},
|
2550
3981
|
'HostConfig' => {
|
3982
|
+
'Binds'=> ["/var/run/docker.sock:/var/run/docker.sock","/usr/bin/docker:/usr/bin/docker" ],
|
2551
3983
|
'PortBindings' => {
|
2552
3984
|
'8888/tcp' => [
|
2553
3985
|
{'HostPort' => "#{port}", 'HostIp' => 'localhost'}
|
2554
3986
|
],
|
3987
|
+
'6006/tcp' => [
|
3988
|
+
{'HostPort' => "#{tensport}", 'HostIp' => 'localhost'}
|
3989
|
+
],
|
2555
3990
|
},
|
2556
3991
|
},
|
2557
3992
|
}
|
@@ -2559,21 +3994,104 @@ module Cnvrg
|
|
2559
3994
|
container.start()
|
2560
3995
|
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
2561
3996
|
container.exec(command, tty: true)
|
2562
|
-
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
3997
|
+
# command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
3998
|
+
# container.exec(command, tty: true)
|
3999
|
+
# command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
|
4000
|
+
# container.exec(command, tty: true)
|
4001
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
|
2563
4002
|
container.exec(command, tty: true)
|
2564
|
-
command = ["/bin/bash", "-lc", "
|
4003
|
+
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
2565
4004
|
container.exec(command, tty: true)
|
2566
|
-
|
4005
|
+
say "#{container.id}:#{port}##{tensport}"
|
4006
|
+
rescue => e
|
4007
|
+
puts e
|
4008
|
+
if e.message.include? "is not running"
|
4009
|
+
return config_remote(image_name, port-1, tensport-1)
|
4010
|
+
end
|
4011
|
+
|
4012
|
+
if container
|
4013
|
+
container.kill()
|
4014
|
+
end
|
4015
|
+
return false
|
4016
|
+
end
|
4017
|
+
end
|
4018
|
+
|
4019
|
+
|
4020
|
+
desc '', '', :hide => true
|
4021
|
+
method_option :login, :type => :string, :aliases => ["-l"], :default => ""
|
4022
|
+
|
4023
|
+
def config_netrc(container)
|
4024
|
+
|
4025
|
+
login_content = options["login"]
|
4026
|
+
|
4027
|
+
container = Docker::Container.get(container)
|
4028
|
+
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
4029
|
+
container.exec(command, tty: true)
|
4030
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
|
4031
|
+
container.exec(command, tty: true)
|
4032
|
+
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
4033
|
+
container.exec(command, tty: true)
|
4034
|
+
say "OK"
|
4035
|
+
|
4036
|
+
end
|
4037
|
+
|
4038
|
+
desc '', '', :hide => true
|
4039
|
+
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
4040
|
+
method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
|
4041
|
+
method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
4042
|
+
|
4043
|
+
|
4044
|
+
def config_remote_gpu(image_name, port=7654, tensport=6006)
|
4045
|
+
local_images = Docker::Image.all
|
4046
|
+
|
4047
|
+
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
|
4048
|
+
if docker_image_local.empty?
|
4049
|
+
say "no image"
|
4050
|
+
exit(1)
|
4051
|
+
end
|
4052
|
+
|
4053
|
+
begin
|
4054
|
+
login_content = options["login"]
|
4055
|
+
app_dir = options["app_dir"]
|
4056
|
+
cmd= options["cmd"]
|
4057
|
+
|
4058
|
+
# image_settings = {
|
4059
|
+
# 'Image' => "#{image_name}:latest",
|
4060
|
+
# 'User' => 'ds',
|
4061
|
+
# 'Cmd' => cmd,
|
4062
|
+
# 'WorkingDir' => app_dir,
|
4063
|
+
# 'ExposedPorts' => {
|
4064
|
+
# '8888/tcp' => {},
|
4065
|
+
# },
|
4066
|
+
# 'HostConfig' => {
|
4067
|
+
# 'PortBindings' => {
|
4068
|
+
# '8888/tcp' => [
|
4069
|
+
# {'HostPort' => "#{port}", 'HostIp' => 'localhost'}
|
4070
|
+
# ],
|
4071
|
+
# '6006/tcp' => [
|
4072
|
+
# {'HostPort' => "6006", 'HostIp' => 'localhost'}
|
4073
|
+
# ],
|
4074
|
+
# },
|
4075
|
+
# },
|
4076
|
+
# }
|
4077
|
+
|
4078
|
+
container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} -v /usr/bin/nvidia-smi:/usr/bin/nvidia-smi -v /var/run/docker.sock:/var/run/docker.sock -v /usr/bin/docker:/usr/bin/docker #{image_name}:latest #{cmd} `
|
4079
|
+
container_id = container_id.gsub("\n", "")
|
4080
|
+
container = Docker::Container.get(container_id)
|
4081
|
+
# container.start()
|
4082
|
+
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
4083
|
+
container.exec(command, tty: true)
|
4084
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
|
2567
4085
|
container.exec(command, tty: true)
|
2568
4086
|
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
2569
4087
|
container.exec(command, tty: true)
|
2570
|
-
say "#{container.id}:#{port}"
|
4088
|
+
say "#{container.id}:#{port}##{tensport}"
|
2571
4089
|
rescue => e
|
2572
4090
|
if e.message.include? "is not running"
|
2573
|
-
|
4091
|
+
puts "running asgain with: #{port-1} #{tensport-1}"
|
4092
|
+
return config_remote_gpu(image_name, port-1, tensport-1)
|
2574
4093
|
end
|
2575
|
-
|
2576
|
-
puts e
|
4094
|
+
|
2577
4095
|
if container
|
2578
4096
|
container.kill()
|
2579
4097
|
end
|
@@ -2581,8 +4099,8 @@ module Cnvrg
|
|
2581
4099
|
end
|
2582
4100
|
end
|
2583
4101
|
|
2584
|
-
desc '', ''
|
2585
|
-
method_option :login, :type => :string, :aliases => ["-l"
|
4102
|
+
desc '', '', :hide => true
|
4103
|
+
method_option :login, :type => :string, :aliases => ["-l"], :default => ""
|
2586
4104
|
|
2587
4105
|
def config_flask_remote(image_name, port=80)
|
2588
4106
|
local_images = Docker::Image.all
|
@@ -2598,7 +4116,7 @@ module Cnvrg
|
|
2598
4116
|
image_settings = {
|
2599
4117
|
'Image' => "#{image_name}:latest",
|
2600
4118
|
'User' => 'ds',
|
2601
|
-
'Cmd' => '/
|
4119
|
+
'Cmd' => '/usr/local/cnvrg/start_super.sh',
|
2602
4120
|
'WorkingDir' => '/home/ds/app',
|
2603
4121
|
'ExposedPorts' => {
|
2604
4122
|
'80/tcp' => {},
|
@@ -2615,21 +4133,54 @@ module Cnvrg
|
|
2615
4133
|
container.start()
|
2616
4134
|
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
2617
4135
|
container.exec(command, tty: true)
|
2618
|
-
command = ["/bin/bash", "-lc", "
|
4136
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
|
2619
4137
|
container.exec(command, tty: true)
|
2620
|
-
command = ["/bin/bash", "-lc", "
|
4138
|
+
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
2621
4139
|
container.exec(command, tty: true)
|
2622
|
-
|
4140
|
+
say "#{container.id}:#{port}"
|
4141
|
+
rescue => e
|
4142
|
+
pus e
|
4143
|
+
if e.message.include? "is not running"
|
4144
|
+
return "port is taken"
|
4145
|
+
end
|
4146
|
+
puts "error"
|
4147
|
+
if container
|
4148
|
+
container.kill()
|
4149
|
+
end
|
4150
|
+
return false
|
4151
|
+
end
|
4152
|
+
end
|
4153
|
+
|
4154
|
+
desc '', '', :hide => true
|
4155
|
+
method_option :login, :type => :string, :aliases => ["-l"], :default => ""
|
4156
|
+
|
4157
|
+
def config_flask_remote_gpu(image_name, port=80)
|
4158
|
+
local_images = Docker::Image.all
|
4159
|
+
|
4160
|
+
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
|
4161
|
+
if docker_image_local.empty?
|
4162
|
+
say "no image"
|
4163
|
+
exit(1)
|
4164
|
+
end
|
4165
|
+
|
4166
|
+
begin
|
4167
|
+
login_content = options["login"]
|
4168
|
+
container_id = `nvidia-docker run -itd -p 80:80 -w /home/ds/app #{image_name}:latest /usr/local/cnvrg/start_super.sh`
|
4169
|
+
container_id = container_id.gsub("\n", "")
|
4170
|
+
container = Docker::Container.get(container_id)
|
4171
|
+
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
4172
|
+
container.exec(command, tty: true)
|
4173
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds:ds /home/ds/.netrc"]
|
2623
4174
|
container.exec(command, tty: true)
|
2624
4175
|
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
2625
4176
|
container.exec(command, tty: true)
|
2626
4177
|
say "#{container.id}:#{port}"
|
2627
4178
|
rescue => e
|
4179
|
+
puts e
|
2628
4180
|
if e.message.include? "is not running"
|
2629
4181
|
return "port is taken"
|
2630
4182
|
end
|
2631
4183
|
puts "error"
|
2632
|
-
puts e
|
2633
4184
|
if container
|
2634
4185
|
container.kill()
|
2635
4186
|
end
|
@@ -2637,7 +4188,7 @@ module Cnvrg
|
|
2637
4188
|
end
|
2638
4189
|
end
|
2639
4190
|
|
2640
|
-
desc '', ''
|
4191
|
+
desc '', '', :hide => true
|
2641
4192
|
|
2642
4193
|
def upload_cnvrg_image(image_name)
|
2643
4194
|
verify_logged_in(false)
|
@@ -2647,12 +4198,12 @@ module Cnvrg
|
|
2647
4198
|
|
2648
4199
|
path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}.zip"
|
2649
4200
|
@files = Cnvrg::Files.new("", "")
|
2650
|
-
python_arr = Cnvrg::Images.get_installed_packages("python")
|
2651
|
-
py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
|
2652
|
-
system_arr = Cnvrg::Images.get_installed_packages("system")
|
2653
|
-
sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
|
2654
|
-
bash_history = Cnvrg::Images.get_bash_history
|
2655
|
-
res = @files.upload_image(path, image_name, owner, true, true,
|
4201
|
+
# python_arr = Cnvrg::Images.get_installed_packages("python")
|
4202
|
+
# py = python_arr.join(",") unless python_arr.nil? or python_arr.empty?
|
4203
|
+
# system_arr = Cnvrg::Images.get_installed_packages("system")
|
4204
|
+
# sys = system_arr.join(",") unless system_arr.nil? or system_arr.empty?
|
4205
|
+
# bash_history = Cnvrg::Images.get_bash_history
|
4206
|
+
res = @files.upload_image(path, image_name, owner, true, true, "", "", "", "Image made by cnvrg.io team", "")
|
2656
4207
|
|
2657
4208
|
end
|
2658
4209
|
|
@@ -2698,7 +4249,7 @@ module Cnvrg
|
|
2698
4249
|
end
|
2699
4250
|
end
|
2700
4251
|
|
2701
|
-
desc '', ''
|
4252
|
+
desc '', '', :hide => true
|
2702
4253
|
|
2703
4254
|
def download_image(image_name, image_slug)
|
2704
4255
|
begin
|
@@ -2713,11 +4264,13 @@ module Cnvrg
|
|
2713
4264
|
if @files.download_image(path, image_slug, owner)
|
2714
4265
|
|
2715
4266
|
dir_path = File.expand_path('~')+"/.cnvrg/tmp/#{image_name}"
|
2716
|
-
FileUtils.
|
4267
|
+
FileUtils.rm_rf([dir_path])
|
4268
|
+
|
2717
4269
|
Zip::File.open(path) do |zip_file|
|
2718
|
-
# Handle entries one by one
|
2719
4270
|
zip_file.each do |entry|
|
4271
|
+
|
2720
4272
|
f_path=File.join(dir_path, entry.name)
|
4273
|
+
FileUtils.mkdir_p(File.dirname(f_path))
|
2721
4274
|
zip_file.extract(entry, f_path)
|
2722
4275
|
end
|
2723
4276
|
end
|
@@ -2755,13 +4308,13 @@ module Cnvrg
|
|
2755
4308
|
owner = Cnvrg::CLI.get_owner()
|
2756
4309
|
res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
|
2757
4310
|
if Cnvrg::CLI.is_response_success(res)
|
2758
|
-
printf "%-20s %-20s %-30s %-20s %-20s
|
4311
|
+
printf "%-20s %-20s %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
|
2759
4312
|
res["result"]["images"].each do |u|
|
2760
4313
|
time = Time.parse(u["created_at"])
|
2761
4314
|
update_at = get_local_time(time)
|
2762
4315
|
created_by = u["created_by"]
|
2763
4316
|
|
2764
|
-
printf "%-20s %-20s %-30s %-20s %-20s
|
4317
|
+
printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
|
2765
4318
|
end
|
2766
4319
|
end
|
2767
4320
|
log_end(0)
|
@@ -2856,22 +4409,33 @@ module Cnvrg
|
|
2856
4409
|
path = download_image(image_name, image["slug"])
|
2857
4410
|
if path
|
2858
4411
|
say "Building image", Thor::Shell::Color::BLUE
|
2859
|
-
Docker.options[:read_timeout]=
|
2860
|
-
image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile', 't' => "#{image_name}:
|
4412
|
+
Docker.options[:read_timeout]=216000
|
4413
|
+
image = Docker::Image.build_from_dir(path, {'dockerfile' => 'Dockerfile.cpu', 't' => "#{image_name}:latest"}) do |v|
|
4414
|
+
begin
|
4415
|
+
if (log = JSON.parse(v)) && log.has_key?("stream")
|
4416
|
+
next if log["stream"].starts_with? "Step"
|
4417
|
+
$stdout.puts log["stream"]
|
4418
|
+
end
|
4419
|
+
rescue
|
4420
|
+
end
|
4421
|
+
|
4422
|
+
end
|
4423
|
+
|
2861
4424
|
if not image.nil?
|
2862
|
-
say "Finished downloading image, cleaning up..", Thor::Shell::Color::GREEN
|
2863
4425
|
FileUtils.rm_rf(path)
|
2864
4426
|
checks = Helpers.checkmark()
|
2865
|
-
say "#{checks}
|
4427
|
+
say "#{checks} Image built successfully", Thor::Shell::Color::GREEN
|
2866
4428
|
log_end(0)
|
2867
4429
|
return image
|
2868
4430
|
log_end(0)
|
2869
4431
|
else
|
4432
|
+
|
2870
4433
|
say "Could not build image", Thor::Shell::Color::RED
|
2871
4434
|
log_end(1, "Could build image")
|
2872
4435
|
return false
|
2873
4436
|
end
|
2874
4437
|
else
|
4438
|
+
|
2875
4439
|
say "Could not download image", Thor::Shell::Color::RED
|
2876
4440
|
log_end(1, "Could build image")
|
2877
4441
|
return false
|
@@ -2899,7 +4463,9 @@ module Cnvrg
|
|
2899
4463
|
#
|
2900
4464
|
# end
|
2901
4465
|
# end
|
2902
|
-
rescue
|
4466
|
+
rescue => e
|
4467
|
+
|
4468
|
+
|
2903
4469
|
say "Couldn't build image", Thor::Shell::Color::RED
|
2904
4470
|
|
2905
4471
|
rescue SignalException
|
@@ -2929,7 +4495,7 @@ module Cnvrg
|
|
2929
4495
|
if docker_image_local.size == 0
|
2930
4496
|
|
2931
4497
|
if yes? "Image wasn't found locally, pull image from cnvrg repository?", Thor::Shell::Color::YELLOW
|
2932
|
-
image =
|
4498
|
+
image = pull(docker_image)
|
2933
4499
|
if image
|
2934
4500
|
say "downloaded image: #{docker_image}"
|
2935
4501
|
@image = Images.new(working_dir, docker_image)
|
@@ -3006,7 +4572,12 @@ module Cnvrg
|
|
3006
4572
|
if url.nil? or url.empty?
|
3007
4573
|
url = "https://cnvrg.io/api"
|
3008
4574
|
end
|
3009
|
-
config =
|
4575
|
+
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
4576
|
+
compression_path = "#{home_dir}/.cnvrg/tmp"
|
4577
|
+
if config and !config.nil? and !config.empty? and !config.to_h[:compression_path].nil?
|
4578
|
+
compression_path = config.to_h[:compression_path]
|
4579
|
+
end
|
4580
|
+
config = {owner: owner, username: username, version_last_check: get_start_day(), api: url,compression_path:compression_path}
|
3010
4581
|
|
3011
4582
|
File.open(home_dir+"/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
3012
4583
|
return true
|
@@ -3060,7 +4631,7 @@ module Cnvrg
|
|
3060
4631
|
|
3061
4632
|
def log_end(exit_status=0, error=nil)
|
3062
4633
|
begin
|
3063
|
-
if exit_status
|
4634
|
+
if exit_status != 0
|
3064
4635
|
$LOG.error exit_status: exit_status, error: error
|
3065
4636
|
else
|
3066
4637
|
$LOG.info exit_status: exit_status
|
@@ -3072,7 +4643,6 @@ module Cnvrg
|
|
3072
4643
|
def self.is_response_success(response, should_exit=true)
|
3073
4644
|
if response.nil?
|
3074
4645
|
if !Cnvrg::Helpers.internet_connection?
|
3075
|
-
# Cnvrg::CLI.log_end(1,"no internet connection")
|
3076
4646
|
say("<%= color('Error:You seems to be offline', RED) %>")
|
3077
4647
|
else
|
3078
4648
|
say("<%= color('Error', RED) %>")
|
@@ -3107,12 +4677,26 @@ module Cnvrg
|
|
3107
4677
|
return owner
|
3108
4678
|
end
|
3109
4679
|
end
|
4680
|
+
|
3110
4681
|
def get_base_url
|
3111
4682
|
home_dir = File.expand_path('~')
|
3112
4683
|
|
3113
4684
|
config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
|
3114
4685
|
api = config.to_h[:api]
|
3115
|
-
return api.gsub!("/api","")
|
4686
|
+
return api.gsub!("/api", "")
|
4687
|
+
end
|
4688
|
+
def get_compression_path
|
4689
|
+
home_dir = File.expand_path('~')
|
4690
|
+
|
4691
|
+
config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
|
4692
|
+
compression_path = config.to_h[:compression_path]
|
4693
|
+
if compression_path.nil?
|
4694
|
+
compression_path = "#{home_dir}/.cnvrg/tmp/"
|
4695
|
+
end
|
4696
|
+
if !compression_path.ends_with? "/"
|
4697
|
+
compression_path = compression_path + "/"
|
4698
|
+
end
|
4699
|
+
return compression_path
|
3116
4700
|
end
|
3117
4701
|
|
3118
4702
|
def get_project_home
|
@@ -3166,7 +4750,6 @@ module Cnvrg
|
|
3166
4750
|
end
|
3167
4751
|
$LOG = LogStashLogger.new(type: :file, path: logfile, sync: true)
|
3168
4752
|
rescue => e
|
3169
|
-
# puts e
|
3170
4753
|
end
|
3171
4754
|
end
|
3172
4755
|
|
@@ -3194,6 +4777,9 @@ module Cnvrg
|
|
3194
4777
|
|
3195
4778
|
config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
|
3196
4779
|
version_date = config.to_h[:version_last_check]
|
4780
|
+
if version_date.nil?
|
4781
|
+
version_date = get_start_day()
|
4782
|
+
end
|
3197
4783
|
next_day = get_start_day+ 86399
|
3198
4784
|
if not (version_date..next_day).cover?(Time.now)
|
3199
4785
|
if should_update_version()
|
@@ -3219,11 +4805,15 @@ module Cnvrg
|
|
3219
4805
|
def is_cnvrg_dir(dir=Dir.pwd)
|
3220
4806
|
current_dir = dir
|
3221
4807
|
home_dir = File.expand_path('~')
|
4808
|
+
if current_dir.eql? home_dir
|
4809
|
+
return false
|
4810
|
+
end
|
3222
4811
|
is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
|
3223
4812
|
until is_cnvrg == true
|
3224
4813
|
current_dir = File.expand_path("..", current_dir)
|
3225
4814
|
is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
|
3226
|
-
if File.expand_path("..", current_dir).eql? home_dir
|
4815
|
+
if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
|
4816
|
+
is_cnvrg = false
|
3227
4817
|
break
|
3228
4818
|
end
|
3229
4819
|
end
|
@@ -3234,6 +4824,19 @@ module Cnvrg
|
|
3234
4824
|
end
|
3235
4825
|
end
|
3236
4826
|
|
4827
|
+
def data_dir_include()
|
4828
|
+
all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
|
4829
|
+
all_dirs.flatten!
|
4830
|
+
all_dirs.each do |a|
|
4831
|
+
if a.include? "/.cnvrg"
|
4832
|
+
ignore = File.dirname(a)
|
4833
|
+
return ignore
|
4834
|
+
end
|
4835
|
+
end
|
4836
|
+
return nil
|
4837
|
+
|
4838
|
+
end
|
4839
|
+
|
3237
4840
|
def verify_software_installed(software)
|
3238
4841
|
begin
|
3239
4842
|
install_url = Cnvrg::CLI::INSTALLATION_URLS[software.to_sym]
|
@@ -3409,8 +5012,151 @@ module Cnvrg
|
|
3409
5012
|
|
3410
5013
|
end
|
3411
5014
|
|
5015
|
+
def update_deleted(deleted)
|
5016
|
+
final = []
|
5017
|
+
deleted.each do |d|
|
5018
|
+
all_subs = d.split("/")
|
5019
|
+
to_add = true
|
5020
|
+
value = all_subs[0]
|
5021
|
+
all_subs.each_with_index do |a, i|
|
5022
|
+
if final.include? value+"/"
|
5023
|
+
to_add = false
|
5024
|
+
break
|
5025
|
+
end
|
5026
|
+
value = value+"/"+all_subs[i+1] if i <all_subs.size-1
|
5027
|
+
end
|
5028
|
+
final << d if to_add
|
5029
|
+
|
5030
|
+
|
5031
|
+
end
|
5032
|
+
|
5033
|
+
|
5034
|
+
return final
|
5035
|
+
end
|
5036
|
+
|
5037
|
+
def get_cmd_path_in_dir(main_dir, sub_dir)
|
5038
|
+
first = Pathname.new main_dir
|
5039
|
+
second = Pathname.new sub_dir
|
5040
|
+
relative = second.relative_path_from first
|
5041
|
+
if relative.eql? "."
|
5042
|
+
return ""
|
5043
|
+
else
|
5044
|
+
return relative
|
5045
|
+
end
|
5046
|
+
end
|
5047
|
+
|
5048
|
+
def create_tar(path_in, path_out, tar_files,no_compression=false)
|
5049
|
+
#The cd is meant for cases when running cnvrg data uplaod not in the main folder
|
5050
|
+
begin
|
5051
|
+
if no_compression
|
5052
|
+
`cd #{path_in} && tar -cf #{path_out} -T #{tar_files}`
|
5053
|
+
else
|
5054
|
+
`cd #{path_in} && tar -czf #{path_out} -T #{tar_files}`
|
5055
|
+
end
|
5056
|
+
rescue => e
|
5057
|
+
puts "Exception while compressing data: #{e.message}"
|
5058
|
+
end
|
5059
|
+
|
5060
|
+
return $?.success?
|
5061
|
+
end
|
5062
|
+
|
5063
|
+
def extarct_tar(file_path, dir_path)
|
5064
|
+
`tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
|
5065
|
+
return $?.success?
|
5066
|
+
end
|
5067
|
+
def cpu_usage
|
5068
|
+
cpu_usage = 0.0
|
5069
|
+
begin
|
5070
|
+
cpu = `top b -n 2 -d 2 |grep %Cpu |tail -1 |awk '{print $2+$3}'`
|
5071
|
+
if !cpu.nil?
|
5072
|
+
cpu_usage = cpu.to_f
|
5073
|
+
end
|
5074
|
+
rescue
|
5075
|
+
cpu_usage = 0.0
|
5076
|
+
end
|
5077
|
+
|
5078
|
+
return cpu_usage
|
5079
|
+
end
|
5080
|
+
def memory_usage
|
5081
|
+
prec = 0.0
|
5082
|
+
begin
|
5083
|
+
used = `free -mt |grep Mem: |awk '{print $3}'`
|
5084
|
+
total = `free -mt |grep Mem: |awk '{print $2}'`
|
5085
|
+
|
5086
|
+
used_f = used.to_f if !used.nil?
|
5087
|
+
total_f = total.to_f if !total.nil?
|
5088
|
+
prec = (used_f / total_f)*100
|
5089
|
+
prec = prec.round(2)
|
5090
|
+
rescue
|
5091
|
+
prec = 0.0
|
5092
|
+
end
|
5093
|
+
return prec
|
5094
|
+
|
5095
|
+
|
5096
|
+
end
|
5097
|
+
def gpu_util
|
5098
|
+
gpu = [0.0, 0.0]
|
5099
|
+
begin
|
5100
|
+
gpu_stats = `nvidia-smi --query-gpu=utilization.gpu,utilization.memory --format=csv |tail -1`
|
5101
|
+
|
5102
|
+
if !gpu_stats.nil?
|
5103
|
+
gpu = gpu_stats.strip
|
5104
|
+
gpu = gpu_stats.gsub!("%","").split(",")
|
5105
|
+
gpu[0] = gpu[0].to_f
|
5106
|
+
gpu[1] = gpu[1].to_f
|
5107
|
+
return gpu
|
5108
|
+
end
|
5109
|
+
|
5110
|
+
rescue
|
5111
|
+
return gpu
|
5112
|
+
end
|
5113
|
+
|
5114
|
+
|
5115
|
+
end
|
5116
|
+
def usage_metrics_in_docker(docker_id)
|
5117
|
+
res = [0.0,0.0]
|
5118
|
+
begin
|
5119
|
+
if docker_id.nil?
|
5120
|
+
docker_id = `cat /etc/hostname`
|
5121
|
+
end
|
5122
|
+
stats = `sudo docker stats #{docker_id} --no-stream --format "{{.CPUPerc}},{{.MemPerc}},{{.BlockIO}}"`
|
5123
|
+
if !stats.nil?
|
5124
|
+
conv = stats.split(",")
|
5125
|
+
cpu = conv[0].gsub!("%","").to_f
|
5126
|
+
memory = conv[1].gsub!("%","").to_f
|
5127
|
+
res = [cpu,memory]
|
5128
|
+
return res
|
5129
|
+
end
|
5130
|
+
rescue
|
5131
|
+
return res
|
5132
|
+
end
|
5133
|
+
|
5134
|
+
|
5135
|
+
|
5136
|
+
end
|
5137
|
+
|
5138
|
+
def with_progress_bar
|
5139
|
+
not_done = true
|
5140
|
+
upload_progress_bar = ProgressBar.create(:title => "Upload progress",
|
5141
|
+
:format => '%a <%B> %p%% %t',
|
5142
|
+
:starting_at => 0,
|
5143
|
+
:total => 600)
|
5144
|
+
pb = Thread.new do
|
5145
|
+
while not_done do
|
5146
|
+
upload_progress_bar.increment
|
5147
|
+
sleep(1)
|
5148
|
+
|
5149
|
+
end
|
5150
|
+
end
|
3412
5151
|
|
5152
|
+
yield.tap do # After yielding to the block, save the return value
|
5153
|
+
not_done = false # Tell the thread to exit, cleaning up after itself…
|
5154
|
+
pb.join # …and wait for it to do so.
|
5155
|
+
end
|
5156
|
+
end
|
3413
5157
|
end
|
5158
|
+
|
5159
|
+
|
3414
5160
|
end
|
3415
5161
|
end
|
3416
5162
|
|