cnvrg 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +9 -5
- data/lib/cnvrg.rb +0 -1
- data/lib/cnvrg/Images.rb +112 -26
- data/lib/cnvrg/api.rb +7 -4
- data/lib/cnvrg/cli.rb +2201 -455
- data/lib/cnvrg/data.rb +72 -0
- data/lib/cnvrg/datafiles.rb +509 -0
- data/lib/cnvrg/dataset.rb +296 -0
- data/lib/cnvrg/experiment.rb +19 -10
- data/lib/cnvrg/files.rb +302 -208
- data/lib/cnvrg/helpers.rb +42 -1
- data/lib/cnvrg/job.rb +0 -1
- data/lib/cnvrg/project.rb +55 -15
- data/lib/cnvrg/ssh.rb +95 -0
- data/lib/cnvrg/version.rb +2 -1
- metadata +48 -17
@@ -0,0 +1,296 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
module Cnvrg
|
3
|
+
class Dataset
|
4
|
+
attr_reader :slug, :owner, :title, :local_path, :working_dir
|
5
|
+
|
6
|
+
RemoteURL ||= "https://cnvrg.io"
|
7
|
+
|
8
|
+
def initialize(project_home)
|
9
|
+
begin
|
10
|
+
config = YAML.load_file(project_home+"/.cnvrg/config.yml")
|
11
|
+
@local_path = project_home
|
12
|
+
@title = config[:dataset_name]
|
13
|
+
@slug = config[:dataset_slug]
|
14
|
+
@owner = config[:owner]
|
15
|
+
@working_dir = project_home
|
16
|
+
rescue => e
|
17
|
+
|
18
|
+
end
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
def last_local_commit
|
23
|
+
idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
|
24
|
+
return idx[:commit]
|
25
|
+
end
|
26
|
+
|
27
|
+
def snapshot
|
28
|
+
commit = last_local_commit
|
29
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
|
30
|
+
CLI.is_response_success(response)
|
31
|
+
return response
|
32
|
+
|
33
|
+
end
|
34
|
+
def list(owner)
|
35
|
+
response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
|
36
|
+
CLI.is_response_success(response)
|
37
|
+
return response
|
38
|
+
|
39
|
+
end
|
40
|
+
def list_commits
|
41
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
|
42
|
+
CLI.is_response_success(response)
|
43
|
+
return response
|
44
|
+
|
45
|
+
end
|
46
|
+
def url
|
47
|
+
url = Cnvrg::Helpers.remote_url
|
48
|
+
"#{url}/#{self.owner}/projects/#{self.slug}"
|
49
|
+
end
|
50
|
+
|
51
|
+
def update_ignore_list(new_ignore)
|
52
|
+
|
53
|
+
if new_ignore.nil? or new_ignore.empty?
|
54
|
+
return true
|
55
|
+
end
|
56
|
+
begin
|
57
|
+
File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
|
58
|
+
f.puts("\n")
|
59
|
+
|
60
|
+
new_ignore.each do |i|
|
61
|
+
f.puts("#{i}\n")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
return true
|
65
|
+
rescue
|
66
|
+
return false
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def get_ignore_list
|
71
|
+
ignore_list = []
|
72
|
+
File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
|
73
|
+
line = line.strip
|
74
|
+
|
75
|
+
if line.start_with? "#" or ignore_list.include? line
|
76
|
+
next
|
77
|
+
end
|
78
|
+
if line.end_with? "/" or File.directory?(line)
|
79
|
+
if line.end_with? "/"
|
80
|
+
ignore_list << line.chop
|
81
|
+
else
|
82
|
+
ignore_list << line
|
83
|
+
end
|
84
|
+
all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
|
85
|
+
ignore_list << all_sub.flatten
|
86
|
+
ignore_list << line
|
87
|
+
else
|
88
|
+
ignore_list << line
|
89
|
+
end
|
90
|
+
end
|
91
|
+
return ignore_list.flatten
|
92
|
+
|
93
|
+
end
|
94
|
+
|
95
|
+
|
96
|
+
def self.init(owner, dataset_name, is_public=false)
|
97
|
+
list_dirs = [".cnvrg"
|
98
|
+
]
|
99
|
+
list_files = [
|
100
|
+
".cnvrgignore",
|
101
|
+
".cnvrg/config.yml"
|
102
|
+
]
|
103
|
+
|
104
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
105
|
+
begin
|
106
|
+
response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
|
107
|
+
Cnvrg::CLI.is_response_success(response)
|
108
|
+
response = JSON.parse response["result"]
|
109
|
+
dataset_slug = response["slug"]
|
110
|
+
|
111
|
+
config = {dataset_name: dataset_name,
|
112
|
+
dataset_slug: dataset_slug,
|
113
|
+
owner: owner}
|
114
|
+
FileUtils.mkdir_p list_dirs
|
115
|
+
FileUtils.touch list_files
|
116
|
+
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
117
|
+
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
|
118
|
+
rescue => e
|
119
|
+
"Exception in init data: #{e.message}"
|
120
|
+
return false
|
121
|
+
end
|
122
|
+
return true
|
123
|
+
end
|
124
|
+
def self.clone(owner, dataset_name,dataset_slug)
|
125
|
+
|
126
|
+
begin
|
127
|
+
list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
|
128
|
+
]
|
129
|
+
list_files = [
|
130
|
+
"#{dataset_name}/.cnvrgignore",
|
131
|
+
"#{dataset_name}/.cnvrg/config.yml"
|
132
|
+
]
|
133
|
+
|
134
|
+
config = {dataset_name: dataset_name,
|
135
|
+
dataset_slug: dataset_slug,
|
136
|
+
owner: owner}
|
137
|
+
|
138
|
+
|
139
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
140
|
+
FileUtils.mkdir_p list_dirs
|
141
|
+
FileUtils.touch list_files
|
142
|
+
File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
143
|
+
File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
|
144
|
+
rescue => e
|
145
|
+
puts "Exception in clone request:#{e.message}"
|
146
|
+
return false
|
147
|
+
end
|
148
|
+
return true
|
149
|
+
end
|
150
|
+
|
151
|
+
def self.init_container(owner, dataset_slug,dataset_name)
|
152
|
+
|
153
|
+
cnvrgignore = Helpers.cnvrgignore_content
|
154
|
+
begin
|
155
|
+
list_dirs = [ ".cnvrg"
|
156
|
+
]
|
157
|
+
list_files = [
|
158
|
+
".cnvrgignore",
|
159
|
+
".cnvrg/config.yml"
|
160
|
+
]
|
161
|
+
FileUtils.mkdir_p list_dirs
|
162
|
+
FileUtils.touch list_files
|
163
|
+
|
164
|
+
config = {dataset_name: dataset_name,
|
165
|
+
dataset_slug: dataset_slug,
|
166
|
+
owner: owner}
|
167
|
+
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
168
|
+
|
169
|
+
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
|
170
|
+
rescue => e
|
171
|
+
return false
|
172
|
+
end
|
173
|
+
return true
|
174
|
+
end
|
175
|
+
|
176
|
+
|
177
|
+
def get_idx
|
178
|
+
YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
179
|
+
end
|
180
|
+
def url
|
181
|
+
url = Cnvrg::Helpers.remote_url
|
182
|
+
"#{url}/#{self.owner}/datasets/#{self.slug}"
|
183
|
+
end
|
184
|
+
def generate_idx
|
185
|
+
if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
|
186
|
+
old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
187
|
+
else
|
188
|
+
old_idx = nil
|
189
|
+
end
|
190
|
+
|
191
|
+
tree_idx = Hash.new(0)
|
192
|
+
|
193
|
+
list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
|
194
|
+
list_ignore = self.get_ignore_list()
|
195
|
+
list.each do |e|
|
196
|
+
label = e.gsub(self.local_path + "/", "")
|
197
|
+
if File.directory? e
|
198
|
+
if list_ignore.include? label
|
199
|
+
next
|
200
|
+
end
|
201
|
+
tree_idx[label+"/"] = nil
|
202
|
+
else
|
203
|
+
if list_ignore.include? label
|
204
|
+
next
|
205
|
+
end
|
206
|
+
sha1 = Digest::SHA1.file(e).hexdigest
|
207
|
+
if old_idx.nil? or old_idx.to_h["tree"].nil?
|
208
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
209
|
+
elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
|
210
|
+
tree_idx[label] = {sha1: sha1, commit_time: nil}
|
211
|
+
else
|
212
|
+
tree_idx[label] = old_idx["tree"][label]
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
|
218
|
+
|
219
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
220
|
+
return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
221
|
+
end
|
222
|
+
def create_volume
|
223
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
|
224
|
+
CLI.is_response_success(response)
|
225
|
+
return response
|
226
|
+
end
|
227
|
+
|
228
|
+
def downlowd_updated_data(current_commit)
|
229
|
+
|
230
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
|
231
|
+
CLI.is_response_success(response,false)
|
232
|
+
return response
|
233
|
+
end
|
234
|
+
def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
|
235
|
+
if local_idx.nil?
|
236
|
+
local_idx = self.generate_idx
|
237
|
+
end
|
238
|
+
|
239
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
|
240
|
+
CLI.is_response_success(response,false)
|
241
|
+
return response
|
242
|
+
end
|
243
|
+
|
244
|
+
|
245
|
+
|
246
|
+
def compare_commits(commit)
|
247
|
+
response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
|
248
|
+
CLI.is_response_success(response,false)
|
249
|
+
return response
|
250
|
+
end
|
251
|
+
|
252
|
+
def compare_commit(commit)
|
253
|
+
if commit.nil? or commit.empty?
|
254
|
+
commit = last_local_commit
|
255
|
+
end
|
256
|
+
response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
|
257
|
+
CLI.is_response_success(response,false)
|
258
|
+
update_is_new_branch(response["result"]["new_branch"])
|
259
|
+
return response["result"]["new_branch"]
|
260
|
+
end
|
261
|
+
|
262
|
+
def update_idx_with_files_commits!(files, commit_time)
|
263
|
+
|
264
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
265
|
+
files.each do |path|
|
266
|
+
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
267
|
+
end
|
268
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
269
|
+
|
270
|
+
return true
|
271
|
+
end
|
272
|
+
def update_idx(idx)
|
273
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
274
|
+
|
275
|
+
return true
|
276
|
+
end
|
277
|
+
|
278
|
+
|
279
|
+
def update_idx_with_commit!(commit)
|
280
|
+
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
281
|
+
idx_hash[:commit] = commit
|
282
|
+
|
283
|
+
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
284
|
+
return true
|
285
|
+
end
|
286
|
+
|
287
|
+
def revert(working_dir)
|
288
|
+
FileUtils.rm_rf working_dir
|
289
|
+
# response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
|
290
|
+
# CLI.is_response_success(response)
|
291
|
+
|
292
|
+
|
293
|
+
end
|
294
|
+
|
295
|
+
end
|
296
|
+
end
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -9,11 +9,11 @@ module Cnvrg
|
|
9
9
|
@slug = nil
|
10
10
|
end
|
11
11
|
|
12
|
-
def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity)
|
12
|
+
def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path)
|
13
13
|
res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
|
14
14
|
{input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
|
15
|
-
title: name, email_notification: email_notification, machine_activity: machine_activity})
|
16
|
-
Cnvrg::CLI.is_response_success(res)
|
15
|
+
title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
|
16
|
+
Cnvrg::CLI.is_response_success(res,false)
|
17
17
|
|
18
18
|
@slug = res.to_h["result"].to_h["slug"]
|
19
19
|
|
@@ -37,7 +37,7 @@ module Cnvrg
|
|
37
37
|
def end_notebook_session(notebook_slug, end_commit)
|
38
38
|
res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
|
39
39
|
{notebook_slug: notebook_slug, end_commit: end_commit})
|
40
|
-
Cnvrg::CLI.is_response_success(res)
|
40
|
+
Cnvrg::CLI.is_response_success(res,false)
|
41
41
|
|
42
42
|
return res
|
43
43
|
|
@@ -78,27 +78,36 @@ module Cnvrg
|
|
78
78
|
|
79
79
|
end
|
80
80
|
|
81
|
-
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
|
81
|
+
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
|
82
82
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
83
83
|
commit_sha1: commit_to_run,
|
84
84
|
instance_type: instance_type,
|
85
85
|
scheduling_query:scheduling_query,
|
86
|
-
local_timestamp:local_timestamp
|
86
|
+
local_timestamp:local_timestamp,
|
87
|
+
grid: grid,
|
88
|
+
path_to_cmd:path_to_cmd,dataset_slug:data,
|
89
|
+
dataset_commit: data_commit})
|
87
90
|
return response
|
88
91
|
end
|
89
92
|
|
90
93
|
def upload_temp_log(temp_log, cpu_average, memory_average)
|
91
94
|
response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
|
92
|
-
exp_slug: @slug
|
93
|
-
|
94
|
-
Cnvrg::CLI.is_response_success(response)
|
95
|
+
exp_slug: @slug})
|
96
|
+
Cnvrg::CLI.is_response_success(response,false)
|
95
97
|
end
|
96
98
|
|
97
99
|
def end(output, exit_status, end_commit, cpu_average, memory_average)
|
100
|
+
#if remote try to remove
|
98
101
|
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
|
99
102
|
exit_status: exit_status, end_commit: end_commit,
|
100
103
|
cpu_average: cpu_average, memory_average: memory_average})
|
101
|
-
Cnvrg::CLI.is_response_success(response)
|
104
|
+
Cnvrg::CLI.is_response_success(response,false)
|
105
|
+
|
106
|
+
begin
|
107
|
+
FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
|
108
|
+
rescue
|
109
|
+
|
110
|
+
end
|
102
111
|
end
|
103
112
|
end
|
104
113
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -1,11 +1,14 @@
|
|
1
1
|
require 'mimemagic'
|
2
2
|
require 'aws-sdk'
|
3
3
|
require 'URLcrypt'
|
4
|
+
require 'tempfile'
|
4
5
|
|
5
6
|
module Cnvrg
|
6
7
|
class Files
|
7
8
|
|
8
|
-
LARGE_FILE=1024*1024*
|
9
|
+
LARGE_FILE=1024*1024*5
|
10
|
+
MULTIPART_SPLIT=10000000
|
11
|
+
|
9
12
|
attr_reader :base_resource
|
10
13
|
|
11
14
|
def initialize(owner, project_slug)
|
@@ -19,9 +22,11 @@ module Cnvrg
|
|
19
22
|
file_size = File.size(absolute_path).to_f
|
20
23
|
mime_type = MimeMagic.by_path(absolute_path)
|
21
24
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
25
|
+
sha1 = Digest::SHA1.file(absolute_path).hexdigest
|
26
|
+
|
22
27
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
23
28
|
commit_sha1: commit_sha1, file_name: file_name,
|
24
|
-
file_size: file_size, file_content_type: content_type})
|
29
|
+
file_size: file_size, file_content_type: content_type, sha1: sha1})
|
25
30
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
26
31
|
path = upload_resp["result"]["path"]
|
27
32
|
if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
|
@@ -29,24 +34,27 @@ module Cnvrg
|
|
29
34
|
else
|
30
35
|
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
31
36
|
end
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
+
return s3_res
|
38
|
+
# if s3_res
|
39
|
+
# update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
|
40
|
+
# blob_id: upload_resp["result"]["id"]})
|
41
|
+
# is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
|
42
|
+
#
|
43
|
+
# return is_suc
|
37
44
|
end
|
38
45
|
return false
|
39
46
|
end
|
40
|
-
|
47
|
+
|
48
|
+
def upload_log_file(absolute_path, relative_path, log_date)
|
41
49
|
file_name = File.basename relative_path
|
42
50
|
file_size = File.size(absolute_path).to_f
|
43
51
|
content_type = "text/x-log"
|
44
52
|
upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
45
|
-
|
46
|
-
|
53
|
+
file_name: file_name, log_date: log_date,
|
54
|
+
file_size: file_size, file_content_type: content_type})
|
47
55
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
48
|
-
|
49
|
-
|
56
|
+
path = upload_resp["result"]["path"]
|
57
|
+
s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
|
50
58
|
end
|
51
59
|
if s3_res
|
52
60
|
return true
|
@@ -54,18 +62,19 @@ module Cnvrg
|
|
54
62
|
return false
|
55
63
|
|
56
64
|
end
|
57
|
-
|
65
|
+
|
66
|
+
def upload_exec_file(absolute_path, image_name, commit_id)
|
58
67
|
file_name = File.basename absolute_path
|
59
68
|
file_size = File.size(absolute_path).to_f
|
60
69
|
content_type = "application/zip"
|
61
70
|
begin
|
62
71
|
upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
|
63
72
|
file_name: file_name,
|
64
|
-
image_name:image_name,
|
73
|
+
image_name: image_name,
|
65
74
|
file_size: file_size,
|
66
75
|
file_content_type: content_type,
|
67
76
|
project_slug: @project_slug,
|
68
|
-
commit_id:commit_id})
|
77
|
+
commit_id: commit_id})
|
69
78
|
# puts upload_resp
|
70
79
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
71
80
|
if upload_resp["result"]["image"] == -1
|
@@ -75,21 +84,20 @@ module Cnvrg
|
|
75
84
|
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
76
85
|
|
77
86
|
end
|
78
|
-
|
79
|
-
|
87
|
+
if s3_res
|
88
|
+
return upload_resp["result"]["id"]
|
89
|
+
end
|
90
|
+
return false
|
91
|
+
rescue SignalException
|
92
|
+
|
93
|
+
say "\nAborting"
|
94
|
+
exit(1)
|
80
95
|
end
|
81
|
-
return false
|
82
|
-
rescue SignalException
|
83
96
|
|
84
|
-
say "\nAborting"
|
85
|
-
exit(1)
|
86
97
|
end
|
87
98
|
|
88
|
-
end
|
89
99
|
|
90
|
-
|
91
|
-
|
92
|
-
def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
|
100
|
+
def upload_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message, commit_id)
|
93
101
|
file_name = File.basename absolute_path
|
94
102
|
file_size = File.size(absolute_path).to_f
|
95
103
|
if is_base
|
@@ -99,40 +107,42 @@ module Cnvrg
|
|
99
107
|
content_type = "application/gzip"
|
100
108
|
end
|
101
109
|
begin
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
if
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
110
|
+
upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
|
111
|
+
file_name: file_name,
|
112
|
+
image_name: image_name,
|
113
|
+
file_size: file_size,
|
114
|
+
file_content_type: content_type,
|
115
|
+
is_public: is_public,
|
116
|
+
project_slug: @project_slug,
|
117
|
+
commit_id: commit_id,
|
118
|
+
dpkg: dpkg,
|
119
|
+
py2: libraries,
|
120
|
+
py3: libraries,
|
121
|
+
|
122
|
+
bash_history: bash,
|
123
|
+
commit_message: message,
|
124
|
+
is_base: is_base})
|
125
|
+
# puts upload_resp
|
126
|
+
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
127
|
+
path = upload_resp["result"]["path"]
|
128
|
+
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
129
|
+
if s3_res
|
130
|
+
commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
|
131
|
+
if Cnvrg::CLI.is_response_success(commit_resp, false)
|
132
|
+
return commit_resp["result"]["image"]
|
133
|
+
else
|
134
|
+
return false
|
135
|
+
end
|
126
136
|
|
137
|
+
end
|
127
138
|
end
|
128
|
-
|
129
|
-
|
130
|
-
rescue =>e
|
131
|
-
puts e
|
139
|
+
return false
|
140
|
+
rescue => e
|
132
141
|
end
|
133
142
|
|
134
143
|
end
|
135
|
-
|
144
|
+
|
145
|
+
def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
|
136
146
|
file_name = File.basename absolute_path
|
137
147
|
file_size = File.size(absolute_path).to_f
|
138
148
|
if is_base
|
@@ -143,16 +153,16 @@ module Cnvrg
|
|
143
153
|
end
|
144
154
|
begin
|
145
155
|
upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
+
file_name: file_name,
|
157
|
+
image_name: image_name,
|
158
|
+
file_size: file_size,
|
159
|
+
file_content_type: content_type,
|
160
|
+
is_public: is_public,
|
161
|
+
dpkg: dpkg,
|
162
|
+
libraries: libraries,
|
163
|
+
bash_history: bash,
|
164
|
+
commit_message: message,
|
165
|
+
is_base: is_base})
|
156
166
|
# puts upload_resp
|
157
167
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
158
168
|
path = upload_resp["result"]["path"]
|
@@ -168,8 +178,7 @@ module Cnvrg
|
|
168
178
|
end
|
169
179
|
end
|
170
180
|
return false
|
171
|
-
rescue =>e
|
172
|
-
puts e
|
181
|
+
rescue => e
|
173
182
|
end
|
174
183
|
|
175
184
|
end
|
@@ -188,7 +197,6 @@ module Cnvrg
|
|
188
197
|
|
189
198
|
return true
|
190
199
|
rescue => e
|
191
|
-
puts e
|
192
200
|
return false
|
193
201
|
end
|
194
202
|
|
@@ -202,174 +210,260 @@ module Cnvrg
|
|
202
210
|
|
203
211
|
def upload_large_files_s3(upload_resp, file_path)
|
204
212
|
begin
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
213
|
+
# speed_uri = URI.parse("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
|
214
|
+
# speed_http_object = Net::HTTP.new(speed_uri.host, speed_uri.port)
|
215
|
+
# speed_http_object.use_ssl = true if speed_uri.scheme == 'https'
|
216
|
+
# speed_request = Net::HTTP::Get.new("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
|
217
|
+
#
|
218
|
+
# speed_body = ""
|
219
|
+
# speed_http_object.start do |http|
|
220
|
+
# response = http.request speed_request
|
221
|
+
# speed_body = response.read_body
|
222
|
+
# end
|
223
|
+
# speed_tmp = Tempfile.new('speed')
|
224
|
+
# speed_tmp << speed_body
|
225
|
+
# speed_tmp.flush
|
226
|
+
# speed_tmp.close
|
227
|
+
# count = 0
|
228
|
+
# sum = 0
|
229
|
+
# speed_res = `python #{speed_tmp.path} --json`
|
230
|
+
# if !speed_res.nil? and !speed_res.empty?
|
231
|
+
# upload = JSON.parse(speed_res)["upload"]
|
232
|
+
# if !upload.nil?
|
233
|
+
# up_spped = (upload/1000000).round(2)
|
234
|
+
# end
|
235
|
+
#
|
236
|
+
#
|
237
|
+
# file_size = File.size(file_path)/1048576
|
238
|
+
# est_up = ((file_size*8)/up_spped)/60
|
239
|
+
# puts est_up
|
240
|
+
# end
|
241
|
+
|
242
|
+
|
243
|
+
sts_path = upload_resp["result"]["path_sts"]
|
244
|
+
s4cmd_path = upload_resp["result"]["path_s4cmd"]
|
245
|
+
|
246
|
+
uri = URI.parse(sts_path)
|
247
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
248
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
249
|
+
request = Net::HTTP::Get.new(sts_path)
|
250
|
+
|
251
|
+
body = ""
|
252
|
+
http_object.start do |http|
|
253
|
+
response = http.request request
|
254
|
+
body = response.read_body
|
255
|
+
end
|
256
|
+
|
257
|
+
URLcrypt::key = [body].pack('H*')
|
258
|
+
is_python = false
|
259
|
+
s4cmd_suc = false
|
260
|
+
s4cmd_install_suc = false
|
261
|
+
# python_version=`python --version > /dev/null 2>&1`; is_python=$?.success?
|
262
|
+
# if is_python
|
263
|
+
#
|
264
|
+
# s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
|
265
|
+
# if !s4cmd_suc
|
266
|
+
# `pip install s4cmd > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
|
267
|
+
# end
|
268
|
+
#
|
269
|
+
# end
|
270
|
+
if !s4cmd_suc and !s4cmd_install_suc
|
271
|
+
s3 = Aws::S3::Resource.new(
|
272
|
+
:access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
|
273
|
+
:secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
|
274
|
+
:session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
|
275
|
+
:region => URLcrypt.decrypt(upload_resp["result"]["region"]))
|
276
|
+
resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
|
277
|
+
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
278
|
+
upload_file(file_path, {:use_accelerate_endpoint => true})
|
279
|
+
else
|
280
|
+
s4cmd_uri = URI.parse(s4cmd_path)
|
281
|
+
s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
|
282
|
+
s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
|
283
|
+
s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
|
284
|
+
|
285
|
+
s4cmd_body = ""
|
286
|
+
s4cmd_http_object.start do |http|
|
287
|
+
response = http.request s4cmd_request
|
288
|
+
s4cmd_body = response.read_body
|
289
|
+
end
|
290
|
+
s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
|
291
|
+
aws_access_key_id=aws_access_key_id,
|
292
|
+
aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
|
293
|
+
aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
|
294
|
+
aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
|
295
|
+
aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
|
296
|
+
|
297
|
+
|
298
|
+
tmp = Tempfile.new('s4cmd.py')
|
299
|
+
tmp << s4cmd_new_body
|
300
|
+
tmp.flush
|
301
|
+
tmp.close
|
302
|
+
#
|
303
|
+
is_success = false
|
304
|
+
count = 0
|
305
|
+
while !is_success and count <3
|
306
|
+
resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
|
307
|
+
is_success =$?.success?
|
308
|
+
count +=1
|
309
|
+
|
310
|
+
end
|
311
|
+
resp= is_success
|
312
|
+
|
313
|
+
end
|
314
|
+
|
315
|
+
return resp
|
316
|
+
|
317
|
+
rescue => e
|
318
|
+
if File.exist? tmp
|
319
|
+
FileUtils.rm_rf [tmp]
|
320
|
+
end
|
321
|
+
return false
|
322
|
+
|
215
323
|
end
|
216
|
-
|
217
|
-
s3 = Aws::S3::Resource.new(
|
218
|
-
:access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
|
219
|
-
:secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
|
220
|
-
:session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
|
221
|
-
:region => URLcrypt.decrypt(upload_resp["result"]["region"]))
|
222
|
-
resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
|
223
|
-
object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
|
224
|
-
upload_file(file_path)
|
225
|
-
return resp
|
226
|
-
rescue =>e
|
227
|
-
puts e
|
228
|
-
return false
|
324
|
+
return true
|
229
325
|
|
230
326
|
end
|
231
|
-
return true
|
232
327
|
|
233
|
-
|
328
|
+
def upload_small_files_s3(url_path, file_path, content_type)
|
329
|
+
url = URI.parse(url_path)
|
330
|
+
file = File.open(file_path, "rb")
|
331
|
+
body = file.read
|
332
|
+
begin
|
333
|
+
Net::HTTP.start(url.host) do |http|
|
334
|
+
http.send_request("PUT", url.request_uri, body, {
|
335
|
+
"content-type" => content_type,
|
336
|
+
})
|
337
|
+
end
|
338
|
+
return true
|
339
|
+
rescue Interrupt
|
340
|
+
return false
|
341
|
+
rescue
|
342
|
+
return false
|
343
|
+
end
|
344
|
+
end
|
234
345
|
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
http.send_request("PUT", url.request_uri, body, {
|
242
|
-
"content-type" => content_type,
|
243
|
-
})
|
346
|
+
def upload_url(file_path)
|
347
|
+
response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
|
348
|
+
if Cnvrg::CLI.is_response_success(response, false)
|
349
|
+
return response
|
350
|
+
else
|
351
|
+
return nil
|
244
352
|
end
|
245
|
-
|
246
|
-
rescue Interrupt
|
247
|
-
return false
|
248
|
-
rescue
|
249
|
-
return false
|
353
|
+
|
250
354
|
end
|
251
|
-
end
|
252
355
|
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
return response
|
257
|
-
else
|
258
|
-
return nil
|
356
|
+
def delete_file(absolute_path, relative_path, commit_sha1)
|
357
|
+
response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
358
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
259
359
|
end
|
260
360
|
|
261
|
-
|
361
|
+
def delete_dir(absolute_path, relative_path, commit_sha1)
|
362
|
+
response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
363
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
364
|
+
end
|
262
365
|
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
366
|
+
def create_dir(absolute_path, relative_path, commit_sha1)
|
367
|
+
response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
368
|
+
return Cnvrg::CLI.is_response_success(response, false)
|
369
|
+
end
|
267
370
|
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
371
|
+
def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
|
372
|
+
begin
|
373
|
+
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
|
374
|
+
Cnvrg::CLI.is_response_success(res, false)
|
375
|
+
if res["result"]
|
376
|
+
download_resp = res
|
377
|
+
filename = download_resp["result"]["filename"]
|
378
|
+
|
379
|
+
absolute_path += ".conflict" if conflict
|
380
|
+
sts_path = download_resp["result"]["path_sts"]
|
381
|
+
uri = URI.parse(sts_path)
|
382
|
+
http_object = Net::HTTP.new(uri.host, uri.port)
|
383
|
+
http_object.use_ssl = true if uri.scheme == 'https'
|
384
|
+
request = Net::HTTP::Get.new(sts_path)
|
385
|
+
|
386
|
+
body = ""
|
387
|
+
http_object.start do |http|
|
388
|
+
response = http.request request
|
389
|
+
body = response.read_body
|
390
|
+
end
|
391
|
+
URLcrypt::key = [body].pack('H*')
|
392
|
+
s3 = Aws::S3::Client.new(
|
393
|
+
:access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
|
394
|
+
:secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
|
395
|
+
:session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
|
396
|
+
:region => URLcrypt.decrypt(download_resp["result"]["region"]))
|
397
|
+
File.open(project_home+"/"+absolute_path, 'wb') do |file|
|
398
|
+
resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
|
399
|
+
key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
|
400
|
+
end
|
401
|
+
return true
|
402
|
+
end
|
272
403
|
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
404
|
+
rescue => e
|
405
|
+
return false
|
406
|
+
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
def download_file(absolute_path, relative_path, project_home, conflict=false)
|
279
411
|
res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
|
280
412
|
Cnvrg::CLI.is_response_success(res, false)
|
281
413
|
if res["result"]
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
sts_path = download_resp["result"]["path_sts"]
|
287
|
-
uri = URI.parse(sts_path)
|
288
|
-
http_object = Net::HTTP.new(uri.host, uri.port)
|
289
|
-
http_object.use_ssl = true if uri.scheme == 'https'
|
290
|
-
request = Net::HTTP::Get.new(sts_path)
|
291
|
-
|
292
|
-
body = ""
|
293
|
-
http_object.start do |http|
|
294
|
-
response = http.request request
|
295
|
-
body = response.read_body
|
296
|
-
end
|
297
|
-
URLcrypt::key = [body].pack('H*')
|
298
|
-
s3 = Aws::S3::Client.new(
|
299
|
-
:access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
|
300
|
-
:secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
|
301
|
-
:session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
|
302
|
-
:region => URLcrypt.decrypt(download_resp["result"]["region"]))
|
303
|
-
|
304
|
-
File.open(absolute_path, 'wb') do |file|
|
305
|
-
resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
|
306
|
-
key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
|
307
|
-
end
|
308
|
-
return true
|
309
|
-
end
|
414
|
+
res = res["result"]
|
415
|
+
return false if res["link"].empty? or res["filename"].empty?
|
416
|
+
filename = res["filename"]
|
417
|
+
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
310
418
|
|
311
|
-
|
312
|
-
|
313
|
-
return false
|
419
|
+
FileUtils.mkdir_p project_home + "/" + file_location
|
420
|
+
filename += ".conflict" if conflict
|
314
421
|
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
if res["result"]
|
321
|
-
res = res["result"]
|
322
|
-
return false if res["link"].empty? or res["filename"].empty?
|
323
|
-
filename = res["filename"]
|
324
|
-
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
325
|
-
|
326
|
-
FileUtils.mkdir_p project_home + "/" + file_location
|
327
|
-
filename += ".conflict" if conflict
|
328
|
-
|
329
|
-
File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
|
330
|
-
file.write open(res["link"]).read
|
422
|
+
File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
|
423
|
+
file.write open(res["link"]).read
|
424
|
+
end
|
425
|
+
else
|
426
|
+
return false
|
331
427
|
end
|
332
|
-
|
333
|
-
return false
|
428
|
+
return true
|
334
429
|
end
|
335
|
-
return true
|
336
|
-
end
|
337
430
|
|
338
|
-
|
339
|
-
|
340
|
-
|
341
|
-
def revoke_download_dir(absolute_path, relative_path, project_home)
|
342
|
-
puts FileUtils.rmtree("#{absolute_path}")
|
343
|
-
end
|
431
|
+
def download_dir(absolute_path, relative_path, project_home)
|
432
|
+
FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
|
433
|
+
end
|
344
434
|
|
345
|
-
|
346
|
-
|
347
|
-
|
435
|
+
def revoke_download_dir(absolute_path, relative_path, project_home)
|
436
|
+
puts FileUtils.rmtree("#{absolute_path}")
|
437
|
+
end
|
348
438
|
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
439
|
+
def revoke_download_file(project_home, absolute_path, filename, conflict=false)
|
440
|
+
begin
|
441
|
+
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
442
|
+
|
443
|
+
filename += ".conflict" if conflict
|
444
|
+
FileUtils.remove("#{file_location}/#{filename}")
|
445
|
+
return true
|
446
|
+
rescue
|
447
|
+
return false
|
448
|
+
end
|
354
449
|
end
|
355
|
-
end
|
356
450
|
|
357
|
-
|
451
|
+
def start_commit(new_branch)
|
358
452
|
|
359
|
-
|
360
|
-
|
361
|
-
|
362
|
-
|
363
|
-
|
453
|
+
response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,
|
454
|
+
username: @owner})
|
455
|
+
Cnvrg::CLI.is_response_success(response,false)
|
456
|
+
return response
|
457
|
+
end
|
364
458
|
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
459
|
+
def end_commit(commit_sha1)
|
460
|
+
response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
|
461
|
+
return response
|
462
|
+
end
|
369
463
|
|
370
|
-
|
371
|
-
|
372
|
-
|
464
|
+
def rollback_commit(commit_sha1)
|
465
|
+
response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
|
466
|
+
Cnvrg::CLI.is_response_success(response, false)
|
467
|
+
end
|
373
468
|
end
|
374
469
|
end
|
375
|
-
end
|