cnvrg 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +18 -8
- data/lib/cnvrg/files.rb +0 -1
- data/lib/cnvrg/project.rb +1 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -2
- data/lib/cnvrg/old_cli.rb +0 -332
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f81b4a0cdd83c85f7b0e4d03895590e6c86e08ad
|
4
|
+
data.tar.gz: 5e1d74a7e299a77461ca37eeb6ba2690643690d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a8ee34350de93e9f43f2b9eae856bdcccaa3ed1cedfa6ba799aba5d78f16f996b69a22894ade1267eaccb07d97cc6bc0c56f06a028255f4eb34c5014fd1f38c9
|
7
|
+
data.tar.gz: 30ce0dd1da9e56fb644b01434219883aeb12fa1d96736ac7edccc971d3789da6df3aa8196b8735d3f28d5b22cfdce51d5346f19f63b9762dc765fc2a247959f0
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -288,13 +288,13 @@ module Cnvrg
|
|
288
288
|
resDir = @files.create_dir(f, relative_path, commit_sha1)
|
289
289
|
if resDir
|
290
290
|
update_count += 1
|
291
|
-
successful_updates<<
|
291
|
+
successful_updates<< relative_path
|
292
292
|
end
|
293
293
|
else
|
294
294
|
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
295
|
if res
|
296
296
|
update_count += 1
|
297
|
-
successful_updates<<
|
297
|
+
successful_updates<< relative_path
|
298
298
|
else
|
299
299
|
@files.rollback_commit(commit_sha1)
|
300
300
|
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
@@ -309,12 +309,12 @@ module Cnvrg
|
|
309
309
|
if relative_path.end_with?("/")
|
310
310
|
if @files.delete_dir(f, relative_path, commit_sha1)
|
311
311
|
update_count += 1
|
312
|
-
successful_updates<<
|
312
|
+
successful_updates<< relative_path
|
313
313
|
end
|
314
314
|
else
|
315
315
|
if @files.delete_file(f, relative_path, commit_sha1)
|
316
316
|
update_count += 1
|
317
|
-
successful_updates<<
|
317
|
+
successful_updates<< relative_path
|
318
318
|
end
|
319
319
|
end
|
320
320
|
end
|
@@ -328,17 +328,27 @@ module Cnvrg
|
|
328
328
|
res = @files.end_commit(commit_sha1)
|
329
329
|
if (Cnvrg::CLI.is_response_success(res, false))
|
330
330
|
# save idx
|
331
|
-
|
331
|
+
begin
|
332
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
332
333
|
|
333
334
|
@project.update_idx_with_commit!(commit_sha1)
|
334
|
-
|
335
|
+
rescue
|
336
|
+
@files.rollback_commit(commit_sha1)
|
337
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
338
|
+
exit(1)
|
339
|
+
|
340
|
+
end
|
341
|
+
|
342
|
+
say "Done", Thor::Shell::Color::BLUE
|
335
343
|
if successful_updates.size >0
|
336
344
|
say "Updated:", Thor::Shell::Color::GREEN
|
337
|
-
|
345
|
+
suc = successful_updates.map {|x| x=Helpers.checkmark() +" "+x}
|
346
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
338
347
|
end
|
339
348
|
if successful_deletions.size >0
|
340
349
|
say "Deleted:", Thor::Shell::Color::GREEN
|
341
|
-
|
350
|
+
del = successful_updates.map {|x| x=Helpers.checkmark() +" "+x}
|
351
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
342
352
|
end
|
343
353
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
344
354
|
else
|
data/lib/cnvrg/files.rb
CHANGED
data/lib/cnvrg/project.rb
CHANGED
@@ -218,7 +218,7 @@ module Cnvrg
|
|
218
218
|
|
219
219
|
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
220
220
|
files.each do |path|
|
221
|
-
idx_hash[:tree][path][:commit_time] = commit_time
|
221
|
+
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
222
222
|
end
|
223
223
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
224
224
|
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -136,7 +136,6 @@ files:
|
|
136
136
|
- lib/cnvrg/experiment.rb
|
137
137
|
- lib/cnvrg/files.rb
|
138
138
|
- lib/cnvrg/helpers.rb
|
139
|
-
- lib/cnvrg/old_cli.rb
|
140
139
|
- lib/cnvrg/project.rb
|
141
140
|
- lib/cnvrg/version.rb
|
142
141
|
homepage: https://cnvrg.io
|
data/lib/cnvrg/old_cli.rb
DELETED
@@ -1,332 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'commander/import'
|
5
|
-
require "open4"
|
6
|
-
require 'netrc'
|
7
|
-
require 'net/http'
|
8
|
-
require 'uri'
|
9
|
-
require 'json'
|
10
|
-
require 'yaml'
|
11
|
-
require 'digest' # sha1
|
12
|
-
|
13
|
-
# DEV VERSION
|
14
|
-
#require 'pry'
|
15
|
-
#
|
16
|
-
|
17
|
-
|
18
|
-
program :name, 'cnvrg'
|
19
|
-
program :version, '0.0.1'
|
20
|
-
program :description, 'cnvrg\'s cli for running scripts'
|
21
|
-
|
22
|
-
command :idx do |c|
|
23
|
-
c.action do |args,options|
|
24
|
-
generate_idx
|
25
|
-
end
|
26
|
-
end
|
27
|
-
command :status do |c|
|
28
|
-
c.action do |args,options|
|
29
|
-
status
|
30
|
-
end
|
31
|
-
end
|
32
|
-
command :run do |c|
|
33
|
-
c.syntax = 'cnvrg run script [options]'
|
34
|
-
c.summary = 'monitor run of script'
|
35
|
-
c.description = 'Runs scripts and sending notification when finished'
|
36
|
-
c.option '--script_args script arguments', String, 'script arguments if exist'
|
37
|
-
c.option '--sms', String, 'set sms if you want to be notify by sms when the run is over'
|
38
|
-
c.option '--email', String, 'set email if you want to be notify by email when the run is over'
|
39
|
-
c.option '--silent', String, 'set silent if you want to run script in non-blocking'
|
40
|
-
c.action do |args, options|
|
41
|
-
is_exist = verify_auth_exist()
|
42
|
-
unless is_exist
|
43
|
-
puts("You have to be logged in to continue. Please login.\n")
|
44
|
-
loged_in = login_process()
|
45
|
-
if !loged_in
|
46
|
-
puts("Error: Couldn't logged you in, please run cnvr login separately")
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
silent = ''
|
51
|
-
if options.silent
|
52
|
-
silent = '&'
|
53
|
-
end
|
54
|
-
script = args[0]
|
55
|
-
if args.length != 1
|
56
|
-
puts "Error: only one script should be listed to run"
|
57
|
-
found = false
|
58
|
-
args.each do |arg|
|
59
|
-
ans = agree("should I run #{arg}?")
|
60
|
-
if ans
|
61
|
-
found = true
|
62
|
-
script = arg
|
63
|
-
break
|
64
|
-
end
|
65
|
-
end
|
66
|
-
if not found
|
67
|
-
exit(0)
|
68
|
-
end
|
69
|
-
end
|
70
|
-
#check if file exist
|
71
|
-
exist = File.file?(script)
|
72
|
-
if not exist
|
73
|
-
puts "Error: Can't find #{script}, try using the full path"
|
74
|
-
exit(0)
|
75
|
-
end
|
76
|
-
puts "Running #{script} with args: #{options.script_args}"
|
77
|
-
body = {try: 'yay'}.to_json
|
78
|
-
puts run(true,body)
|
79
|
-
pid, stdin, stdout, stderr = Open4::popen4 "sh"
|
80
|
-
stdin.puts "./#{script} #{options.script_args}"
|
81
|
-
stdin.close
|
82
|
-
|
83
|
-
ignored, status = Process::waitpid2 pid
|
84
|
-
|
85
|
-
#try to run
|
86
|
-
if stderr.read.strip.include? "Permission denied"
|
87
|
-
ans = agree("Your script #{script} isn't executable, should I make it executable?")
|
88
|
-
if ans
|
89
|
-
puts "Making #{script} executable"
|
90
|
-
run_result = %x(chmod +x ./#{script} 2>&1)
|
91
|
-
if run_result.include? "Operation not permitted"
|
92
|
-
puts "Can't make script executable,you should first run sudo chmod +x #{script}"
|
93
|
-
exit(0)
|
94
|
-
end
|
95
|
-
puts "Running again #{script} with args: #{options.script_args}"
|
96
|
-
|
97
|
-
pid, stdin, stdout, stderr = Open4::popen4 "sh"
|
98
|
-
stdin.puts "./#{script} #{options.script_args}"
|
99
|
-
stdin.close
|
100
|
-
ignored, status = Process::waitpid2 pid
|
101
|
-
|
102
|
-
if !stderr.read.strip.empty?
|
103
|
-
puts "Error: #{stderr.read.strip}"
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
puts stdout.read.strip
|
108
|
-
# send results to the server
|
109
|
-
puts "Finished running with status #{status.exitstatus}"
|
110
|
-
puts run(false, body)
|
111
|
-
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
command :new do |c|
|
116
|
-
c.syntax = "cnvrg new project-name"
|
117
|
-
c.summary = "Create a new data science project"
|
118
|
-
|
119
|
-
c.action do |args, options|
|
120
|
-
is_exist = verify_auth_exist()
|
121
|
-
unless is_exist
|
122
|
-
"You have to login"
|
123
|
-
else
|
124
|
-
if args[0].to_s.size == 0
|
125
|
-
exit(0)
|
126
|
-
else
|
127
|
-
new(args[0])
|
128
|
-
link_dir(args[0], "")
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
command :login do |c|
|
135
|
-
c.syntax = 'cnvrg login'
|
136
|
-
c.summary = 'login to cnvrg app'
|
137
|
-
c.description = 'Login to the CNVRG system'
|
138
|
-
c.option '--reset', String, 'set reset if you want to change current login credentials'
|
139
|
-
c.action do |args, options|
|
140
|
-
n = Netrc.read
|
141
|
-
user, pass = n["cnvrg.io"]
|
142
|
-
if user and pass
|
143
|
-
relogin = agree("You are already authenticated with email: #{user}, do you want to login again? (This will delete you're current login credentials")
|
144
|
-
if !relogin
|
145
|
-
exit(0)
|
146
|
-
end
|
147
|
-
end
|
148
|
-
if options.reset
|
149
|
-
n["cnvrg.io"] = "", ""
|
150
|
-
|
151
|
-
end
|
152
|
-
login_process()
|
153
|
-
|
154
|
-
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
def login_process()
|
159
|
-
n = Netrc.read
|
160
|
-
|
161
|
-
email = ask("Please enter your email")
|
162
|
-
email_regex = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
|
163
|
-
is_valid = email_regex.match(email)
|
164
|
-
while !is_valid do
|
165
|
-
email = ask("Please enter a valid email address:")
|
166
|
-
is_valid = email_regex.match(email)
|
167
|
-
end
|
168
|
-
|
169
|
-
password = ask("Please enter your password: ") { |q| q.echo = false }
|
170
|
-
#send API to login
|
171
|
-
token = sign_in(email, password)
|
172
|
-
count = 0
|
173
|
-
while token.nil? and count<3 do
|
174
|
-
do_retry = agree("Wrong email or password, try again?")
|
175
|
-
if do_retry
|
176
|
-
count +=1
|
177
|
-
email = ask("Please enter your email")
|
178
|
-
password = ask("Please enter your password: ") { |q| q.echo = false }
|
179
|
-
token = sign_in(email, password)
|
180
|
-
end
|
181
|
-
end
|
182
|
-
if token.nil?
|
183
|
-
puts "Error: Too many sign in retries"
|
184
|
-
exit(0)
|
185
|
-
else
|
186
|
-
|
187
|
-
|
188
|
-
n.new_item_prefix = "# This entry was added automatically by CNVRG\n"
|
189
|
-
n["cnvrg.io"] = email, token
|
190
|
-
n.save
|
191
|
-
puts "Great, you're in."
|
192
|
-
return true
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
def sign_in(email, password)
|
197
|
-
url = URI.parse("http://localhost:3000/api/v1/users/sign_in")
|
198
|
-
|
199
|
-
req = Net::HTTP::Post.new(url.path)
|
200
|
-
req.add_field("EMAIL", email)
|
201
|
-
req.add_field("PASSWORD", password)
|
202
|
-
|
203
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
204
|
-
http.request(req)
|
205
|
-
end
|
206
|
-
|
207
|
-
result = JSON.parse(response.body)
|
208
|
-
if result["status"] == 200
|
209
|
-
return result["token"]
|
210
|
-
else
|
211
|
-
return nil
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
def verify_auth_exist
|
216
|
-
n = Netrc.read
|
217
|
-
user, pass = n["cnvrg.io"]
|
218
|
-
if user.nil? or pass.nil?
|
219
|
-
puts "Please login first, using \"cnvrg login\" command"
|
220
|
-
return false
|
221
|
-
end
|
222
|
-
return true
|
223
|
-
end
|
224
|
-
|
225
|
-
def get_logged_token
|
226
|
-
n = Netrc.read
|
227
|
-
user, token = n["cnvrg.io"]
|
228
|
-
return token
|
229
|
-
|
230
|
-
end
|
231
|
-
def run(is_start, body)
|
232
|
-
token = get_logged_token()
|
233
|
-
if is_start
|
234
|
-
url = URI.parse("http://localhost:3000/api/v1/cli/start_run")
|
235
|
-
else
|
236
|
-
url = URI.parse("http://localhost:3000/api/v1/cli/end_run")
|
237
|
-
end
|
238
|
-
|
239
|
-
|
240
|
-
req = Net::HTTP::Post.new(url.path)
|
241
|
-
req.add_field("AUTH_TOKEN", token)
|
242
|
-
req.body = body
|
243
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
244
|
-
http.request(req)
|
245
|
-
end
|
246
|
-
|
247
|
-
result = JSON.parse(response.body)
|
248
|
-
puts result
|
249
|
-
end
|
250
|
-
|
251
|
-
def new(project_name)
|
252
|
-
begin
|
253
|
-
`mkdir -p #{project_name} #{project_name}/data #{project_name}/models #{project_name}/notebooks #{project_name}/src #{project_name}/src/data #{project_name}/src/features #{project_name}/src/models #{project_name}/src/visualizations #{project_name}/.cnvrg`
|
254
|
-
`touch #{project_name}/README.md`
|
255
|
-
`touch #{project_name}/.cnvrgignore`
|
256
|
-
rescue
|
257
|
-
"Error"
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def link_dir(project_name, user, host="cnvrg.io")
|
262
|
-
`touch #{project_name}/.cnvrg/project`
|
263
|
-
`echo "owner: #{user}\nproject_slug: #{project_name}\nhost: #{host}" >> #{project_name}/.cnvrg/project`
|
264
|
-
end
|
265
|
-
|
266
|
-
def download
|
267
|
-
|
268
|
-
status_output = status
|
269
|
-
downloadables = status_output["updated_on_server"].merge(status_output["deleted"])
|
270
|
-
|
271
|
-
|
272
|
-
downloadables.each do |d|
|
273
|
-
if d[-1] == "/"
|
274
|
-
# make dir
|
275
|
-
else
|
276
|
-
files << #download(d)
|
277
|
-
end
|
278
|
-
end
|
279
|
-
|
280
|
-
def upload
|
281
|
-
|
282
|
-
end
|
283
|
-
|
284
|
-
def generate_idx
|
285
|
-
|
286
|
-
idx = Hash.new(0)
|
287
|
-
list = Dir['**/**']
|
288
|
-
|
289
|
-
# remove files that are in CNVRGIGNORE
|
290
|
-
# list.select { |x| ! cnvrgignore.include? x }
|
291
|
-
|
292
|
-
list.each do |e|
|
293
|
-
if File.directory? e
|
294
|
-
idx[e+"/"] = nil
|
295
|
-
else
|
296
|
-
idx[e] = { sha1: Digest::SHA1.hexdigest(File.read(e)),
|
297
|
-
commit_time: nil }
|
298
|
-
end
|
299
|
-
end
|
300
|
-
|
301
|
-
File.open("hello/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
302
|
-
|
303
|
-
|
304
|
-
end
|
305
|
-
|
306
|
-
def status
|
307
|
-
url = URI.parse("http://localhost:3000/api/v1/users/yochze/projects/porject/status")
|
308
|
-
token = get_logged_token()
|
309
|
-
load_idx = YAML.load(File.read("hello/.cnvrg/idx.yml"))
|
310
|
-
req = Net::HTTP::Get.new(url, 'Content-Type' => 'application/json')
|
311
|
-
req.add_field("AUTH_TOKEN", token)
|
312
|
-
req.body = { snapshot: load_idx }.to_json
|
313
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
314
|
-
http.request(req)
|
315
|
-
end
|
316
|
-
|
317
|
-
result = JSON.parse(response.body)
|
318
|
-
|
319
|
-
result["status"]["added"].each do |a|
|
320
|
-
puts "A:\t#{a}"
|
321
|
-
end
|
322
|
-
|
323
|
-
result["status"]["updated_on_server"].each do |a|
|
324
|
-
puts "M:\t#{a}"
|
325
|
-
end
|
326
|
-
|
327
|
-
result["status"]["deleted"].each do |a|
|
328
|
-
puts "D:\t#{a}"
|
329
|
-
end
|
330
|
-
|
331
|
-
return result["status"]
|
332
|
-
end
|