cnvrg 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,296 @@
1
+ require 'fileutils'
2
+ module Cnvrg
3
+ class Dataset
4
+ attr_reader :slug, :owner, :title, :local_path, :working_dir
5
+
6
+ RemoteURL ||= "https://cnvrg.io"
7
+
8
+ def initialize(project_home)
9
+ begin
10
+ config = YAML.load_file(project_home+"/.cnvrg/config.yml")
11
+ @local_path = project_home
12
+ @title = config[:dataset_name]
13
+ @slug = config[:dataset_slug]
14
+ @owner = config[:owner]
15
+ @working_dir = project_home
16
+ rescue => e
17
+
18
+ end
19
+
20
+ end
21
+
22
+ def last_local_commit
23
+ idx = YAML.load_file(@local_path + "/.cnvrg/idx.yml")
24
+ return idx[:commit]
25
+ end
26
+
27
+ def snapshot
28
+ commit = last_local_commit
29
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST', {data_commit: commit})
30
+ CLI.is_response_success(response)
31
+ return response
32
+
33
+ end
34
+ def list(owner)
35
+ response = Cnvrg::API.request("users/#{owner}/datasets/list", 'GET')
36
+ CLI.is_response_success(response)
37
+ return response
38
+
39
+ end
40
+ def list_commits
41
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/list_commits", 'GET')
42
+ CLI.is_response_success(response)
43
+ return response
44
+
45
+ end
46
+ def url
47
+ url = Cnvrg::Helpers.remote_url
48
+ "#{url}/#{self.owner}/projects/#{self.slug}"
49
+ end
50
+
51
+ def update_ignore_list(new_ignore)
52
+
53
+ if new_ignore.nil? or new_ignore.empty?
54
+ return true
55
+ end
56
+ begin
57
+ File.open(self.local_path+"/.cnvrgignore", "a+") do |f|
58
+ f.puts("\n")
59
+
60
+ new_ignore.each do |i|
61
+ f.puts("#{i}\n")
62
+ end
63
+ end
64
+ return true
65
+ rescue
66
+ return false
67
+ end
68
+ end
69
+
70
+ def get_ignore_list
71
+ ignore_list = []
72
+ File.open(self.local_path+"/.cnvrgignore", "r").each_line do |line|
73
+ line = line.strip
74
+
75
+ if line.start_with? "#" or ignore_list.include? line
76
+ next
77
+ end
78
+ if line.end_with? "/" or File.directory?(line)
79
+ if line.end_with? "/"
80
+ ignore_list << line.chop
81
+ else
82
+ ignore_list << line
83
+ end
84
+ all_sub = Dir.glob("#{line}/**/*", File::FNM_DOTMATCH).flatten
85
+ ignore_list << all_sub.flatten
86
+ ignore_list << line
87
+ else
88
+ ignore_list << line
89
+ end
90
+ end
91
+ return ignore_list.flatten
92
+
93
+ end
94
+
95
+
96
+ def self.init(owner, dataset_name, is_public=false)
97
+ list_dirs = [".cnvrg"
98
+ ]
99
+ list_files = [
100
+ ".cnvrgignore",
101
+ ".cnvrg/config.yml"
102
+ ]
103
+
104
+ cnvrgignore = Helpers.cnvrgignore_content
105
+ begin
106
+ response = Cnvrg::API.request("cli/create_dataset", 'POST', {title: dataset_name, owner: owner, is_public: is_public})
107
+ Cnvrg::CLI.is_response_success(response)
108
+ response = JSON.parse response["result"]
109
+ dataset_slug = response["slug"]
110
+
111
+ config = {dataset_name: dataset_name,
112
+ dataset_slug: dataset_slug,
113
+ owner: owner}
114
+ FileUtils.mkdir_p list_dirs
115
+ FileUtils.touch list_files
116
+ File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
117
+ File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
118
+ rescue => e
119
+ "Exception in init data: #{e.message}"
120
+ return false
121
+ end
122
+ return true
123
+ end
124
+ def self.clone(owner, dataset_name,dataset_slug)
125
+
126
+ begin
127
+ list_dirs = [ dataset_name, "#{dataset_name}/.cnvrg"
128
+ ]
129
+ list_files = [
130
+ "#{dataset_name}/.cnvrgignore",
131
+ "#{dataset_name}/.cnvrg/config.yml"
132
+ ]
133
+
134
+ config = {dataset_name: dataset_name,
135
+ dataset_slug: dataset_slug,
136
+ owner: owner}
137
+
138
+
139
+ cnvrgignore = Helpers.cnvrgignore_content
140
+ FileUtils.mkdir_p list_dirs
141
+ FileUtils.touch list_files
142
+ File.open("#{dataset_name}/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
143
+ File.open("#{dataset_name}/.cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
144
+ rescue => e
145
+ puts "Exception in clone request:#{e.message}"
146
+ return false
147
+ end
148
+ return true
149
+ end
150
+
151
+ def self.init_container(owner, dataset_slug,dataset_name)
152
+
153
+ cnvrgignore = Helpers.cnvrgignore_content
154
+ begin
155
+ list_dirs = [ ".cnvrg"
156
+ ]
157
+ list_files = [
158
+ ".cnvrgignore",
159
+ ".cnvrg/config.yml"
160
+ ]
161
+ FileUtils.mkdir_p list_dirs
162
+ FileUtils.touch list_files
163
+
164
+ config = {dataset_name: dataset_name,
165
+ dataset_slug: dataset_slug,
166
+ owner: owner}
167
+ File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
168
+
169
+ File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
170
+ rescue => e
171
+ return false
172
+ end
173
+ return true
174
+ end
175
+
176
+
177
+ def get_idx
178
+ YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
179
+ end
180
+ def url
181
+ url = Cnvrg::Helpers.remote_url
182
+ "#{url}/#{self.owner}/datasets/#{self.slug}"
183
+ end
184
+ def generate_idx
185
+ if File.exists? "#{self.local_path}/.cnvrg/idx.yml"
186
+ old_idx = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
187
+ else
188
+ old_idx = nil
189
+ end
190
+
191
+ tree_idx = Hash.new(0)
192
+
193
+ list = Dir.glob("#{self.local_path}/**/**", File::FNM_DOTMATCH).reject { |x| (x =~ /\/\.{1,2}$/) or (x =~ /^#{self.local_path}\/\.cnvrg\/*/) }
194
+ list_ignore = self.get_ignore_list()
195
+ list.each do |e|
196
+ label = e.gsub(self.local_path + "/", "")
197
+ if File.directory? e
198
+ if list_ignore.include? label
199
+ next
200
+ end
201
+ tree_idx[label+"/"] = nil
202
+ else
203
+ if list_ignore.include? label
204
+ next
205
+ end
206
+ sha1 = Digest::SHA1.file(e).hexdigest
207
+ if old_idx.nil? or old_idx.to_h["tree"].nil?
208
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
209
+ elsif old_idx["tree"][label].nil? or old_idx["tree"][label]["sha1"] != sha1
210
+ tree_idx[label] = {sha1: sha1, commit_time: nil}
211
+ else
212
+ tree_idx[label] = old_idx["tree"][label]
213
+ end
214
+ end
215
+ end
216
+
217
+ idx = {commit: old_idx.to_h[:commit], tree: tree_idx}
218
+
219
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
220
+ return YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
221
+ end
222
+ def create_volume
223
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/volumes/create", 'POST')
224
+ CLI.is_response_success(response)
225
+ return response
226
+ end
227
+
228
+ def downlowd_updated_data(current_commit)
229
+
230
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/downlowd_updated_data", 'POST', {current_commit: current_commit})
231
+ CLI.is_response_success(response,false)
232
+ return response
233
+ end
234
+ def compare_idx(new_branch, commit=last_local_commit,local_idx=nil)
235
+ if local_idx.nil?
236
+ local_idx = self.generate_idx
237
+ end
238
+
239
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/status", 'POST', {idx: local_idx, new_branch: new_branch, current_commit: commit})
240
+ CLI.is_response_success(response,false)
241
+ return response
242
+ end
243
+
244
+
245
+
246
+ def compare_commits(commit)
247
+ response = Cnvrg::API.request("users/#{self.owner}/datasets/#{self.slug}/compare_commits", 'POST', {compare_commit: commit,current_commit:last_local_commit})
248
+ CLI.is_response_success(response,false)
249
+ return response
250
+ end
251
+
252
+ def compare_commit(commit)
253
+ if commit.nil? or commit.empty?
254
+ commit = last_local_commit
255
+ end
256
+ response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/commit/compare", 'POST', {current_commit: commit})
257
+ CLI.is_response_success(response,false)
258
+ update_is_new_branch(response["result"]["new_branch"])
259
+ return response["result"]["new_branch"]
260
+ end
261
+
262
+ def update_idx_with_files_commits!(files, commit_time)
263
+
264
+ idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
265
+ files.each do |path|
266
+ idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
267
+ end
268
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
269
+
270
+ return true
271
+ end
272
+ def update_idx(idx)
273
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
274
+
275
+ return true
276
+ end
277
+
278
+
279
+ def update_idx_with_commit!(commit)
280
+ idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
281
+ idx_hash[:commit] = commit
282
+
283
+ File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
284
+ return true
285
+ end
286
+
287
+ def revert(working_dir)
288
+ FileUtils.rm_rf working_dir
289
+ # response = Cnvrg::API.request("users/#{self.owner}/projects/#{self.slug}/revert", 'GET')
290
+ # CLI.is_response_success(response)
291
+
292
+
293
+ end
294
+
295
+ end
296
+ end
@@ -9,11 +9,11 @@ module Cnvrg
9
9
  @slug = nil
10
10
  end
11
11
 
12
- def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity)
12
+ def start(input, platform, machine_name, start_commit, name, email_notification, machine_activity,script_path)
13
13
  res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
14
14
  {input: input, platform: platform, machine_name: machine_name, start_commit: start_commit,
15
- title: name, email_notification: email_notification, machine_activity: machine_activity})
16
- Cnvrg::CLI.is_response_success(res)
15
+ title: name, email_notification: email_notification, machine_activity: machine_activity,script_path:script_path})
16
+ Cnvrg::CLI.is_response_success(res,false)
17
17
 
18
18
  @slug = res.to_h["result"].to_h["slug"]
19
19
 
@@ -37,7 +37,7 @@ module Cnvrg
37
37
  def end_notebook_session(notebook_slug, end_commit)
38
38
  res = Cnvrg::API.request(@base_resource + "notebook/end_session", 'POST',
39
39
  {notebook_slug: notebook_slug, end_commit: end_commit})
40
- Cnvrg::CLI.is_response_success(res)
40
+ Cnvrg::CLI.is_response_success(res,false)
41
41
 
42
42
  return res
43
43
 
@@ -78,27 +78,36 @@ module Cnvrg
78
78
 
79
79
  end
80
80
 
81
- def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
81
+ def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid,path_to_cmd,data, data_commit)
82
82
  response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
83
83
  commit_sha1: commit_to_run,
84
84
  instance_type: instance_type,
85
85
  scheduling_query:scheduling_query,
86
- local_timestamp:local_timestamp})
86
+ local_timestamp:local_timestamp,
87
+ grid: grid,
88
+ path_to_cmd:path_to_cmd,dataset_slug:data,
89
+ dataset_commit: data_commit})
87
90
  return response
88
91
  end
89
92
 
90
93
  def upload_temp_log(temp_log, cpu_average, memory_average)
91
94
  response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', {output: temp_log,
92
- exp_slug: @slug, cpu_average: cpu_average,
93
- memory_average: memory_average})
94
- Cnvrg::CLI.is_response_success(response)
95
+ exp_slug: @slug})
96
+ Cnvrg::CLI.is_response_success(response,false)
95
97
  end
96
98
 
97
99
  def end(output, exit_status, end_commit, cpu_average, memory_average)
100
+ #if remote try to remove
98
101
  response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
99
102
  exit_status: exit_status, end_commit: end_commit,
100
103
  cpu_average: cpu_average, memory_average: memory_average})
101
- Cnvrg::CLI.is_response_success(response)
104
+ Cnvrg::CLI.is_response_success(response,false)
105
+
106
+ begin
107
+ FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
108
+ rescue
109
+
110
+ end
102
111
  end
103
112
  end
104
113
  end
@@ -1,11 +1,14 @@
1
1
  require 'mimemagic'
2
2
  require 'aws-sdk'
3
3
  require 'URLcrypt'
4
+ require 'tempfile'
4
5
 
5
6
  module Cnvrg
6
7
  class Files
7
8
 
8
- LARGE_FILE=1024*1024*100
9
+ LARGE_FILE=1024*1024*5
10
+ MULTIPART_SPLIT=10000000
11
+
9
12
  attr_reader :base_resource
10
13
 
11
14
  def initialize(owner, project_slug)
@@ -19,9 +22,11 @@ module Cnvrg
19
22
  file_size = File.size(absolute_path).to_f
20
23
  mime_type = MimeMagic.by_path(absolute_path)
21
24
  content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
25
+ sha1 = Digest::SHA1.file(absolute_path).hexdigest
26
+
22
27
  upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
23
28
  commit_sha1: commit_sha1, file_name: file_name,
24
- file_size: file_size, file_content_type: content_type})
29
+ file_size: file_size, file_content_type: content_type, sha1: sha1})
25
30
  if Cnvrg::CLI.is_response_success(upload_resp, false)
26
31
  path = upload_resp["result"]["path"]
27
32
  if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
@@ -29,24 +34,27 @@ module Cnvrg
29
34
  else
30
35
  s3_res = upload_small_files_s3(path, absolute_path, content_type)
31
36
  end
32
- if s3_res
33
- Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
34
- blob_id: upload_resp["result"]["id"]})
35
- return true
36
- end
37
+ return s3_res
38
+ # if s3_res
39
+ # update_s3_resp = Cnvrg::API.request(@base_resource + "update_s3", 'POST', {path: path, commit_id: upload_resp["result"]["commit_id"],
40
+ # blob_id: upload_resp["result"]["id"]})
41
+ # is_suc = Cnvrg::CLI.is_response_success(update_s3_resp, false)
42
+ #
43
+ # return is_suc
37
44
  end
38
45
  return false
39
46
  end
40
- def upload_log_file(absolute_path, relative_path,log_date)
47
+
48
+ def upload_log_file(absolute_path, relative_path, log_date)
41
49
  file_name = File.basename relative_path
42
50
  file_size = File.size(absolute_path).to_f
43
51
  content_type = "text/x-log"
44
52
  upload_resp = Cnvrg::API.request("/users/#{@owner}/" + "upload_cli_log", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
45
- file_name: file_name,log_date:log_date,
46
- file_size: file_size, file_content_type: content_type})
53
+ file_name: file_name, log_date: log_date,
54
+ file_size: file_size, file_content_type: content_type})
47
55
  if Cnvrg::CLI.is_response_success(upload_resp, false)
48
- path = upload_resp["result"]["path"]
49
- s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
56
+ path = upload_resp["result"]["path"]
57
+ s3_res = upload_small_files_s3(path, absolute_path, "text/plain")
50
58
  end
51
59
  if s3_res
52
60
  return true
@@ -54,18 +62,19 @@ module Cnvrg
54
62
  return false
55
63
 
56
64
  end
57
- def upload_exec_file(absolute_path,image_name,commit_id)
65
+
66
+ def upload_exec_file(absolute_path, image_name, commit_id)
58
67
  file_name = File.basename absolute_path
59
68
  file_size = File.size(absolute_path).to_f
60
69
  content_type = "application/zip"
61
70
  begin
62
71
  upload_resp = Cnvrg::API.request("users/#{@owner}/images/" + "upload_config", 'POST_FILE', {relative_path: absolute_path,
63
72
  file_name: file_name,
64
- image_name:image_name,
73
+ image_name: image_name,
65
74
  file_size: file_size,
66
75
  file_content_type: content_type,
67
76
  project_slug: @project_slug,
68
- commit_id:commit_id})
77
+ commit_id: commit_id})
69
78
  # puts upload_resp
70
79
  if Cnvrg::CLI.is_response_success(upload_resp, false)
71
80
  if upload_resp["result"]["image"] == -1
@@ -75,21 +84,20 @@ module Cnvrg
75
84
  s3_res = upload_small_files_s3(path, absolute_path, content_type)
76
85
 
77
86
  end
78
- if s3_res
79
- return upload_resp["result"]["id"]
87
+ if s3_res
88
+ return upload_resp["result"]["id"]
89
+ end
90
+ return false
91
+ rescue SignalException
92
+
93
+ say "\nAborting"
94
+ exit(1)
80
95
  end
81
- return false
82
- rescue SignalException
83
96
 
84
- say "\nAborting"
85
- exit(1)
86
97
  end
87
98
 
88
- end
89
99
 
90
-
91
-
92
- def upload_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message,commit_id)
100
+ def upload_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message, commit_id)
93
101
  file_name = File.basename absolute_path
94
102
  file_size = File.size(absolute_path).to_f
95
103
  if is_base
@@ -99,40 +107,42 @@ module Cnvrg
99
107
  content_type = "application/gzip"
100
108
  end
101
109
  begin
102
- upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload", 'POST_FILE', {relative_path: absolute_path,
103
- file_name: file_name,
104
- image_name: image_name,
105
- file_size: file_size,
106
- file_content_type: content_type,
107
- is_public: is_public,
108
- project_slug: @project_slug,
109
- commit_id:commit_id ,
110
- dpkg: dpkg,
111
- libraries: libraries,
112
- bash_history: bash,
113
- commit_message:message,
114
- is_base: is_base})
115
- # puts upload_resp
116
- if Cnvrg::CLI.is_response_success(upload_resp, false)
117
- path = upload_resp["result"]["path"]
118
- s3_res = upload_small_files_s3(path, absolute_path, content_type)
119
- if s3_res
120
- commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
121
- if Cnvrg::CLI.is_response_success(commit_resp, false)
122
- return commit_resp["result"]["image"]
123
- else
124
- return false
125
- end
110
+ upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
111
+ file_name: file_name,
112
+ image_name: image_name,
113
+ file_size: file_size,
114
+ file_content_type: content_type,
115
+ is_public: is_public,
116
+ project_slug: @project_slug,
117
+ commit_id: commit_id,
118
+ dpkg: dpkg,
119
+ py2: libraries,
120
+ py3: libraries,
121
+
122
+ bash_history: bash,
123
+ commit_message: message,
124
+ is_base: is_base})
125
+ # puts upload_resp
126
+ if Cnvrg::CLI.is_response_success(upload_resp, false)
127
+ path = upload_resp["result"]["path"]
128
+ s3_res = upload_small_files_s3(path, absolute_path, content_type)
129
+ if s3_res
130
+ commit_resp = Cnvrg::API.request("users/#{owner}/images/#{upload_resp["result"]["id"]}/" + "commit", 'GET')
131
+ if Cnvrg::CLI.is_response_success(commit_resp, false)
132
+ return commit_resp["result"]["image"]
133
+ else
134
+ return false
135
+ end
126
136
 
137
+ end
127
138
  end
128
- end
129
- return false
130
- rescue =>e
131
- puts e
139
+ return false
140
+ rescue => e
132
141
  end
133
142
 
134
143
  end
135
- def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base,dpkg,libraries,bash,message)
144
+
145
+ def upload_cnvrg_image(absolute_path, image_name, owner, is_public, is_base, dpkg, libraries, bash, message)
136
146
  file_name = File.basename absolute_path
137
147
  file_size = File.size(absolute_path).to_f
138
148
  if is_base
@@ -143,16 +153,16 @@ module Cnvrg
143
153
  end
144
154
  begin
145
155
  upload_resp = Cnvrg::API.request("users/#{owner}/images/" + "upload_cnvrg", 'POST_FILE', {relative_path: absolute_path,
146
- file_name: file_name,
147
- image_name: image_name,
148
- file_size: file_size,
149
- file_content_type: content_type,
150
- is_public: is_public,
151
- dpkg: dpkg,
152
- libraries: libraries,
153
- bash_history: bash,
154
- commit_message:message,
155
- is_base: is_base})
156
+ file_name: file_name,
157
+ image_name: image_name,
158
+ file_size: file_size,
159
+ file_content_type: content_type,
160
+ is_public: is_public,
161
+ dpkg: dpkg,
162
+ libraries: libraries,
163
+ bash_history: bash,
164
+ commit_message: message,
165
+ is_base: is_base})
156
166
  # puts upload_resp
157
167
  if Cnvrg::CLI.is_response_success(upload_resp, false)
158
168
  path = upload_resp["result"]["path"]
@@ -168,8 +178,7 @@ module Cnvrg
168
178
  end
169
179
  end
170
180
  return false
171
- rescue =>e
172
- puts e
181
+ rescue => e
173
182
  end
174
183
 
175
184
  end
@@ -188,7 +197,6 @@ module Cnvrg
188
197
 
189
198
  return true
190
199
  rescue => e
191
- puts e
192
200
  return false
193
201
  end
194
202
 
@@ -202,174 +210,260 @@ module Cnvrg
202
210
 
203
211
  def upload_large_files_s3(upload_resp, file_path)
204
212
  begin
205
- sts_path = upload_resp["result"]["path_sts"]
206
- uri = URI.parse(sts_path)
207
- http_object = Net::HTTP.new(uri.host, uri.port)
208
- http_object.use_ssl = true if uri.scheme == 'https'
209
- request = Net::HTTP::Get.new(sts_path)
210
-
211
- body = ""
212
- http_object.start do |http|
213
- response = http.request request
214
- body = response.read_body
213
+ # speed_uri = URI.parse("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
214
+ # speed_http_object = Net::HTTP.new(speed_uri.host, speed_uri.port)
215
+ # speed_http_object.use_ssl = true if speed_uri.scheme == 'https'
216
+ # speed_request = Net::HTTP::Get.new("https://s3-us-west-2.amazonaws.com/cnvrgw2-dev/speedtest-cli")
217
+ #
218
+ # speed_body = ""
219
+ # speed_http_object.start do |http|
220
+ # response = http.request speed_request
221
+ # speed_body = response.read_body
222
+ # end
223
+ # speed_tmp = Tempfile.new('speed')
224
+ # speed_tmp << speed_body
225
+ # speed_tmp.flush
226
+ # speed_tmp.close
227
+ # count = 0
228
+ # sum = 0
229
+ # speed_res = `python #{speed_tmp.path} --json`
230
+ # if !speed_res.nil? and !speed_res.empty?
231
+ # upload = JSON.parse(speed_res)["upload"]
232
+ # if !upload.nil?
233
+ # up_spped = (upload/1000000).round(2)
234
+ # end
235
+ #
236
+ #
237
+ # file_size = File.size(file_path)/1048576
238
+ # est_up = ((file_size*8)/up_spped)/60
239
+ # puts est_up
240
+ # end
241
+
242
+
243
+ sts_path = upload_resp["result"]["path_sts"]
244
+ s4cmd_path = upload_resp["result"]["path_s4cmd"]
245
+
246
+ uri = URI.parse(sts_path)
247
+ http_object = Net::HTTP.new(uri.host, uri.port)
248
+ http_object.use_ssl = true if uri.scheme == 'https'
249
+ request = Net::HTTP::Get.new(sts_path)
250
+
251
+ body = ""
252
+ http_object.start do |http|
253
+ response = http.request request
254
+ body = response.read_body
255
+ end
256
+
257
+ URLcrypt::key = [body].pack('H*')
258
+ is_python = false
259
+ s4cmd_suc = false
260
+ s4cmd_install_suc = false
261
+ # python_version=`python --version > /dev/null 2>&1`; is_python=$?.success?
262
+ # if is_python
263
+ #
264
+ # s4cmd=`pip freeze 2>/dev/null |grep -e s4cmd -e boto3 > /dev/null 2>&1`; s4cmd_suc=$?.success?
265
+ # if !s4cmd_suc
266
+ # `pip install s4cmd > /dev/null 2>&1`; s4cmd_install_suc=$?.success?
267
+ # end
268
+ #
269
+ # end
270
+ if !s4cmd_suc and !s4cmd_install_suc
271
+ s3 = Aws::S3::Resource.new(
272
+ :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
273
+ :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
274
+ :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
275
+ :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
276
+ resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
277
+ object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
278
+ upload_file(file_path, {:use_accelerate_endpoint => true})
279
+ else
280
+ s4cmd_uri = URI.parse(s4cmd_path)
281
+ s4cmd_http_object = Net::HTTP.new(s4cmd_uri.host, s4cmd_uri.port)
282
+ s4cmd_http_object.use_ssl = true if s4cmd_uri.scheme == 'https'
283
+ s4cmd_request = Net::HTTP::Get.new(s4cmd_path)
284
+
285
+ s4cmd_body = ""
286
+ s4cmd_http_object.start do |http|
287
+ response = http.request s4cmd_request
288
+ s4cmd_body = response.read_body
289
+ end
290
+ s4cmd_new_body = s4cmd_body.gsub(" self.client = self.boto3.client('s3',
291
+ aws_access_key_id=aws_access_key_id,
292
+ aws_secret_access_key=aws_secret_access_key)", " self.client = self.boto3.client('s3',
293
+ aws_access_key_id='#{ URLcrypt.decrypt(upload_resp["result"]["sts_a"])}',
294
+ aws_secret_access_key='#{URLcrypt.decrypt(upload_resp["result"]["sts_s"])}',
295
+ aws_session_token='#{URLcrypt.decrypt(upload_resp["result"]["sts_st"])}')")
296
+
297
+
298
+ tmp = Tempfile.new('s4cmd.py')
299
+ tmp << s4cmd_new_body
300
+ tmp.flush
301
+ tmp.close
302
+ #
303
+ is_success = false
304
+ count = 0
305
+ while !is_success and count <3
306
+ resp = `python #{tmp.path} --num-threads=8 --max-singlepart-upload-size=#{MULTIPART_SPLIT} put -f #{file_path} s3://#{URLcrypt.decrypt(upload_resp["result"]["bucket"])}/#{upload_resp["result"]["path"]+"/"+File.basename(file_path)} > /dev/null 2>&1 `
307
+ is_success =$?.success?
308
+ count +=1
309
+
310
+ end
311
+ resp= is_success
312
+
313
+ end
314
+
315
+ return resp
316
+
317
+ rescue => e
318
+ if File.exist? tmp
319
+ FileUtils.rm_rf [tmp]
320
+ end
321
+ return false
322
+
215
323
  end
216
- URLcrypt::key = [body].pack('H*')
217
- s3 = Aws::S3::Resource.new(
218
- :access_key_id => URLcrypt.decrypt(upload_resp["result"]["sts_a"]),
219
- :secret_access_key => URLcrypt.decrypt(upload_resp["result"]["sts_s"]),
220
- :session_token => URLcrypt.decrypt(upload_resp["result"]["sts_st"]),
221
- :region => URLcrypt.decrypt(upload_resp["result"]["region"]))
222
- resp = s3.bucket(URLcrypt.decrypt(upload_resp["result"]["bucket"])).
223
- object(upload_resp["result"]["path"]+"/"+File.basename(file_path)).
224
- upload_file(file_path)
225
- return resp
226
- rescue =>e
227
- puts e
228
- return false
324
+ return true
229
325
 
230
326
  end
231
- return true
232
327
 
233
- end
328
+ def upload_small_files_s3(url_path, file_path, content_type)
329
+ url = URI.parse(url_path)
330
+ file = File.open(file_path, "rb")
331
+ body = file.read
332
+ begin
333
+ Net::HTTP.start(url.host) do |http|
334
+ http.send_request("PUT", url.request_uri, body, {
335
+ "content-type" => content_type,
336
+ })
337
+ end
338
+ return true
339
+ rescue Interrupt
340
+ return false
341
+ rescue
342
+ return false
343
+ end
344
+ end
234
345
 
235
- def upload_small_files_s3(url_path, file_path, content_type)
236
- url = URI.parse(url_path)
237
- file = File.open(file_path, "rb")
238
- body = file.read
239
- begin
240
- Net::HTTP.start(url.host) do |http|
241
- http.send_request("PUT", url.request_uri, body, {
242
- "content-type" => content_type,
243
- })
346
+ def upload_url(file_path)
347
+ response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
348
+ if Cnvrg::CLI.is_response_success(response, false)
349
+ return response
350
+ else
351
+ return nil
244
352
  end
245
- return true
246
- rescue Interrupt
247
- return false
248
- rescue
249
- return false
353
+
250
354
  end
251
- end
252
355
 
253
- def upload_url(file_path)
254
- response = Cnvrg::API.request(@base_resource + "upload_url", 'POST', {file_s3_path: file_path})
255
- if Cnvrg::CLI.is_response_success(response, false)
256
- return response
257
- else
258
- return nil
356
+ def delete_file(absolute_path, relative_path, commit_sha1)
357
+ response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
358
+ return Cnvrg::CLI.is_response_success(response, false)
259
359
  end
260
360
 
261
- end
361
+ def delete_dir(absolute_path, relative_path, commit_sha1)
362
+ response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
363
+ return Cnvrg::CLI.is_response_success(response, false)
364
+ end
262
365
 
263
- def delete_file(absolute_path, relative_path, commit_sha1)
264
- response = Cnvrg::API.request(@base_resource + "delete_file", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
265
- return Cnvrg::CLI.is_response_success(response, false)
266
- end
366
+ def create_dir(absolute_path, relative_path, commit_sha1)
367
+ response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
368
+ return Cnvrg::CLI.is_response_success(response, false)
369
+ end
267
370
 
268
- def delete_dir(absolute_path, relative_path, commit_sha1)
269
- response = Cnvrg::API.request(@base_resource + "delete_dir", 'DELETE', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
270
- return Cnvrg::CLI.is_response_success(response, false)
271
- end
371
+ def download_file_s3(absolute_path, relative_path, project_home, commit_sha1=nil, conflict=false)
372
+ begin
373
+ res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
374
+ Cnvrg::CLI.is_response_success(res, false)
375
+ if res["result"]
376
+ download_resp = res
377
+ filename = download_resp["result"]["filename"]
378
+
379
+ absolute_path += ".conflict" if conflict
380
+ sts_path = download_resp["result"]["path_sts"]
381
+ uri = URI.parse(sts_path)
382
+ http_object = Net::HTTP.new(uri.host, uri.port)
383
+ http_object.use_ssl = true if uri.scheme == 'https'
384
+ request = Net::HTTP::Get.new(sts_path)
385
+
386
+ body = ""
387
+ http_object.start do |http|
388
+ response = http.request request
389
+ body = response.read_body
390
+ end
391
+ URLcrypt::key = [body].pack('H*')
392
+ s3 = Aws::S3::Client.new(
393
+ :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
394
+ :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
395
+ :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
396
+ :region => URLcrypt.decrypt(download_resp["result"]["region"]))
397
+ File.open(project_home+"/"+absolute_path, 'wb') do |file|
398
+ resp = s3.get_object({bucket: URLcrypt.decrypt(download_resp["result"]["bucket"]),
399
+ key: URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
400
+ end
401
+ return true
402
+ end
272
403
 
273
- def create_dir(absolute_path, relative_path, commit_sha1)
274
- response = Cnvrg::API.request(@base_resource + "create_dir", 'POST', {absolute_path: absolute_path, relative_path: relative_path, commit_sha1: commit_sha1})
275
- return Cnvrg::CLI.is_response_success(response, false)
276
- end
277
- def download_file_s3(absolute_path, relative_path, project_home, conflict=false)
278
- begin
404
+ rescue => e
405
+ return false
406
+
407
+ end
408
+ end
409
+
410
+ def download_file(absolute_path, relative_path, project_home, conflict=false)
279
411
  res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
280
412
  Cnvrg::CLI.is_response_success(res, false)
281
413
  if res["result"]
282
- download_resp = res
283
- filename = download_resp["result"]["filename"]
284
-
285
- absolute_path += ".conflict" if conflict
286
- sts_path = download_resp["result"]["path_sts"]
287
- uri = URI.parse(sts_path)
288
- http_object = Net::HTTP.new(uri.host, uri.port)
289
- http_object.use_ssl = true if uri.scheme == 'https'
290
- request = Net::HTTP::Get.new(sts_path)
291
-
292
- body = ""
293
- http_object.start do |http|
294
- response = http.request request
295
- body = response.read_body
296
- end
297
- URLcrypt::key = [body].pack('H*')
298
- s3 = Aws::S3::Client.new(
299
- :access_key_id => URLcrypt.decrypt(download_resp["result"]["sts_a"]),
300
- :secret_access_key => URLcrypt.decrypt(download_resp["result"]["sts_s"]),
301
- :session_token => URLcrypt.decrypt(download_resp["result"]["sts_st"]),
302
- :region => URLcrypt.decrypt(download_resp["result"]["region"]))
303
-
304
- File.open(absolute_path, 'wb') do |file|
305
- resp = s3.get_object({ bucket:URLcrypt.decrypt(download_resp["result"]["bucket"]),
306
- key:URLcrypt.decrypt(download_resp["result"]["key"])}, target: file)
307
- end
308
- return true
309
- end
414
+ res = res["result"]
415
+ return false if res["link"].empty? or res["filename"].empty?
416
+ filename = res["filename"]
417
+ file_location = absolute_path.gsub(/#{filename}\/?$/, "")
310
418
 
311
- rescue =>e
312
- puts e
313
- return false
419
+ FileUtils.mkdir_p project_home + "/" + file_location
420
+ filename += ".conflict" if conflict
314
421
 
315
- end
316
- end
317
- def download_file(absolute_path, relative_path, project_home, conflict=false)
318
- res = Cnvrg::API.request(@base_resource + "download_file", 'POST', {absolute_path: absolute_path, relative_path: relative_path})
319
- Cnvrg::CLI.is_response_success(res, false)
320
- if res["result"]
321
- res = res["result"]
322
- return false if res["link"].empty? or res["filename"].empty?
323
- filename = res["filename"]
324
- file_location = absolute_path.gsub(/#{filename}\/?$/, "")
325
-
326
- FileUtils.mkdir_p project_home + "/" + file_location
327
- filename += ".conflict" if conflict
328
-
329
- File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
330
- file.write open(res["link"]).read
422
+ File.open("#{project_home}/#{file_location}/#{filename}", "wb") do |file|
423
+ file.write open(res["link"]).read
424
+ end
425
+ else
426
+ return false
331
427
  end
332
- else
333
- return false
428
+ return true
334
429
  end
335
- return true
336
- end
337
430
 
338
- def download_dir(absolute_path, relative_path, project_home)
339
- FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
340
- end
341
- def revoke_download_dir(absolute_path, relative_path, project_home)
342
- puts FileUtils.rmtree("#{absolute_path}")
343
- end
431
+ def download_dir(absolute_path, relative_path, project_home)
432
+ FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
433
+ end
344
434
 
345
- def revoke_download_file(project_home,absolute_path,filename,conflict=false)
346
- begin
347
- file_location = absolute_path.gsub(/#{filename}\/?$/, "")
435
+ def revoke_download_dir(absolute_path, relative_path, project_home)
436
+ puts FileUtils.rmtree("#{absolute_path}")
437
+ end
348
438
 
349
- filename += ".conflict" if conflict
350
- FileUtils.remove("#{file_location}/#{filename}")
351
- return true
352
- rescue
353
- return false
439
+ def revoke_download_file(project_home, absolute_path, filename, conflict=false)
440
+ begin
441
+ file_location = absolute_path.gsub(/#{filename}\/?$/, "")
442
+
443
+ filename += ".conflict" if conflict
444
+ FileUtils.remove("#{file_location}/#{filename}")
445
+ return true
446
+ rescue
447
+ return false
448
+ end
354
449
  end
355
- end
356
450
 
357
- def start_commit(new_branch)
451
+ def start_commit(new_branch)
358
452
 
359
- response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug,new_branch:new_branch,
360
- username: @owner})
361
- Cnvrg::CLI.is_response_success(response)
362
- return response
363
- end
453
+ response = Cnvrg::API.request("#{base_resource}/commit/start", 'POST', {project_slug: @project_slug, new_branch: new_branch,
454
+ username: @owner})
455
+ Cnvrg::CLI.is_response_success(response,false)
456
+ return response
457
+ end
364
458
 
365
- def end_commit(commit_sha1)
366
- response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
367
- return response
368
- end
459
+ def end_commit(commit_sha1)
460
+ response = Cnvrg::API.request("#{base_resource}/commit/end", 'POST', {commit_sha1: commit_sha1})
461
+ return response
462
+ end
369
463
 
370
- def rollback_commit(commit_sha1)
371
- response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
372
- Cnvrg::CLI.is_response_success(response, false)
464
+ def rollback_commit(commit_sha1)
465
+ response = Cnvrg::API.request("#{base_resource}/commit/rollback", 'POST', {commit_sha1: commit_sha1})
466
+ Cnvrg::CLI.is_response_success(response, false)
467
+ end
373
468
  end
374
469
  end
375
- end