cnvrg 0.0.2 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +18 -8
- data/lib/cnvrg/files.rb +0 -1
- data/lib/cnvrg/project.rb +1 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +1 -2
- data/lib/cnvrg/old_cli.rb +0 -332
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f81b4a0cdd83c85f7b0e4d03895590e6c86e08ad
|
4
|
+
data.tar.gz: 5e1d74a7e299a77461ca37eeb6ba2690643690d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a8ee34350de93e9f43f2b9eae856bdcccaa3ed1cedfa6ba799aba5d78f16f996b69a22894ade1267eaccb07d97cc6bc0c56f06a028255f4eb34c5014fd1f38c9
|
7
|
+
data.tar.gz: 30ce0dd1da9e56fb644b01434219883aeb12fa1d96736ac7edccc971d3789da6df3aa8196b8735d3f28d5b22cfdce51d5346f19f63b9762dc765fc2a247959f0
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -288,13 +288,13 @@ module Cnvrg
|
|
288
288
|
resDir = @files.create_dir(f, relative_path, commit_sha1)
|
289
289
|
if resDir
|
290
290
|
update_count += 1
|
291
|
-
successful_updates<<
|
291
|
+
successful_updates<< relative_path
|
292
292
|
end
|
293
293
|
else
|
294
294
|
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
295
|
if res
|
296
296
|
update_count += 1
|
297
|
-
successful_updates<<
|
297
|
+
successful_updates<< relative_path
|
298
298
|
else
|
299
299
|
@files.rollback_commit(commit_sha1)
|
300
300
|
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
@@ -309,12 +309,12 @@ module Cnvrg
|
|
309
309
|
if relative_path.end_with?("/")
|
310
310
|
if @files.delete_dir(f, relative_path, commit_sha1)
|
311
311
|
update_count += 1
|
312
|
-
successful_updates<<
|
312
|
+
successful_updates<< relative_path
|
313
313
|
end
|
314
314
|
else
|
315
315
|
if @files.delete_file(f, relative_path, commit_sha1)
|
316
316
|
update_count += 1
|
317
|
-
successful_updates<<
|
317
|
+
successful_updates<< relative_path
|
318
318
|
end
|
319
319
|
end
|
320
320
|
end
|
@@ -328,17 +328,27 @@ module Cnvrg
|
|
328
328
|
res = @files.end_commit(commit_sha1)
|
329
329
|
if (Cnvrg::CLI.is_response_success(res, false))
|
330
330
|
# save idx
|
331
|
-
|
331
|
+
begin
|
332
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
332
333
|
|
333
334
|
@project.update_idx_with_commit!(commit_sha1)
|
334
|
-
|
335
|
+
rescue
|
336
|
+
@files.rollback_commit(commit_sha1)
|
337
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
338
|
+
exit(1)
|
339
|
+
|
340
|
+
end
|
341
|
+
|
342
|
+
say "Done", Thor::Shell::Color::BLUE
|
335
343
|
if successful_updates.size >0
|
336
344
|
say "Updated:", Thor::Shell::Color::GREEN
|
337
|
-
|
345
|
+
suc = successful_updates.map {|x| x=Helpers.checkmark() +" "+x}
|
346
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
338
347
|
end
|
339
348
|
if successful_deletions.size >0
|
340
349
|
say "Deleted:", Thor::Shell::Color::GREEN
|
341
|
-
|
350
|
+
del = successful_updates.map {|x| x=Helpers.checkmark() +" "+x}
|
351
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
342
352
|
end
|
343
353
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
344
354
|
else
|
data/lib/cnvrg/files.rb
CHANGED
data/lib/cnvrg/project.rb
CHANGED
@@ -218,7 +218,7 @@ module Cnvrg
|
|
218
218
|
|
219
219
|
idx_hash = YAML.load_file("#{self.local_path}/.cnvrg/idx.yml")
|
220
220
|
files.each do |path|
|
221
|
-
idx_hash[:tree][path][:commit_time] = commit_time
|
221
|
+
idx_hash[:tree].to_h[path].to_h[:commit_time] = commit_time
|
222
222
|
end
|
223
223
|
File.open("#{self.local_path}/.cnvrg/idx.yml", 'w') { |f| f.write idx_hash.to_yaml }
|
224
224
|
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -136,7 +136,6 @@ files:
|
|
136
136
|
- lib/cnvrg/experiment.rb
|
137
137
|
- lib/cnvrg/files.rb
|
138
138
|
- lib/cnvrg/helpers.rb
|
139
|
-
- lib/cnvrg/old_cli.rb
|
140
139
|
- lib/cnvrg/project.rb
|
141
140
|
- lib/cnvrg/version.rb
|
142
141
|
homepage: https://cnvrg.io
|
data/lib/cnvrg/old_cli.rb
DELETED
@@ -1,332 +0,0 @@
|
|
1
|
-
#!/usr/bin/env ruby
|
2
|
-
|
3
|
-
require 'rubygems'
|
4
|
-
require 'commander/import'
|
5
|
-
require "open4"
|
6
|
-
require 'netrc'
|
7
|
-
require 'net/http'
|
8
|
-
require 'uri'
|
9
|
-
require 'json'
|
10
|
-
require 'yaml'
|
11
|
-
require 'digest' # sha1
|
12
|
-
|
13
|
-
# DEV VERSION
|
14
|
-
#require 'pry'
|
15
|
-
#
|
16
|
-
|
17
|
-
|
18
|
-
program :name, 'cnvrg'
|
19
|
-
program :version, '0.0.1'
|
20
|
-
program :description, 'cnvrg\'s cli for running scripts'
|
21
|
-
|
22
|
-
command :idx do |c|
|
23
|
-
c.action do |args,options|
|
24
|
-
generate_idx
|
25
|
-
end
|
26
|
-
end
|
27
|
-
command :status do |c|
|
28
|
-
c.action do |args,options|
|
29
|
-
status
|
30
|
-
end
|
31
|
-
end
|
32
|
-
command :run do |c|
|
33
|
-
c.syntax = 'cnvrg run script [options]'
|
34
|
-
c.summary = 'monitor run of script'
|
35
|
-
c.description = 'Runs scripts and sending notification when finished'
|
36
|
-
c.option '--script_args script arguments', String, 'script arguments if exist'
|
37
|
-
c.option '--sms', String, 'set sms if you want to be notify by sms when the run is over'
|
38
|
-
c.option '--email', String, 'set email if you want to be notify by email when the run is over'
|
39
|
-
c.option '--silent', String, 'set silent if you want to run script in non-blocking'
|
40
|
-
c.action do |args, options|
|
41
|
-
is_exist = verify_auth_exist()
|
42
|
-
unless is_exist
|
43
|
-
puts("You have to be logged in to continue. Please login.\n")
|
44
|
-
loged_in = login_process()
|
45
|
-
if !loged_in
|
46
|
-
puts("Error: Couldn't logged you in, please run cnvr login separately")
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
silent = ''
|
51
|
-
if options.silent
|
52
|
-
silent = '&'
|
53
|
-
end
|
54
|
-
script = args[0]
|
55
|
-
if args.length != 1
|
56
|
-
puts "Error: only one script should be listed to run"
|
57
|
-
found = false
|
58
|
-
args.each do |arg|
|
59
|
-
ans = agree("should I run #{arg}?")
|
60
|
-
if ans
|
61
|
-
found = true
|
62
|
-
script = arg
|
63
|
-
break
|
64
|
-
end
|
65
|
-
end
|
66
|
-
if not found
|
67
|
-
exit(0)
|
68
|
-
end
|
69
|
-
end
|
70
|
-
#check if file exist
|
71
|
-
exist = File.file?(script)
|
72
|
-
if not exist
|
73
|
-
puts "Error: Can't find #{script}, try using the full path"
|
74
|
-
exit(0)
|
75
|
-
end
|
76
|
-
puts "Running #{script} with args: #{options.script_args}"
|
77
|
-
body = {try: 'yay'}.to_json
|
78
|
-
puts run(true,body)
|
79
|
-
pid, stdin, stdout, stderr = Open4::popen4 "sh"
|
80
|
-
stdin.puts "./#{script} #{options.script_args}"
|
81
|
-
stdin.close
|
82
|
-
|
83
|
-
ignored, status = Process::waitpid2 pid
|
84
|
-
|
85
|
-
#try to run
|
86
|
-
if stderr.read.strip.include? "Permission denied"
|
87
|
-
ans = agree("Your script #{script} isn't executable, should I make it executable?")
|
88
|
-
if ans
|
89
|
-
puts "Making #{script} executable"
|
90
|
-
run_result = %x(chmod +x ./#{script} 2>&1)
|
91
|
-
if run_result.include? "Operation not permitted"
|
92
|
-
puts "Can't make script executable,you should first run sudo chmod +x #{script}"
|
93
|
-
exit(0)
|
94
|
-
end
|
95
|
-
puts "Running again #{script} with args: #{options.script_args}"
|
96
|
-
|
97
|
-
pid, stdin, stdout, stderr = Open4::popen4 "sh"
|
98
|
-
stdin.puts "./#{script} #{options.script_args}"
|
99
|
-
stdin.close
|
100
|
-
ignored, status = Process::waitpid2 pid
|
101
|
-
|
102
|
-
if !stderr.read.strip.empty?
|
103
|
-
puts "Error: #{stderr.read.strip}"
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
puts stdout.read.strip
|
108
|
-
# send results to the server
|
109
|
-
puts "Finished running with status #{status.exitstatus}"
|
110
|
-
puts run(false, body)
|
111
|
-
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
command :new do |c|
|
116
|
-
c.syntax = "cnvrg new project-name"
|
117
|
-
c.summary = "Create a new data science project"
|
118
|
-
|
119
|
-
c.action do |args, options|
|
120
|
-
is_exist = verify_auth_exist()
|
121
|
-
unless is_exist
|
122
|
-
"You have to login"
|
123
|
-
else
|
124
|
-
if args[0].to_s.size == 0
|
125
|
-
exit(0)
|
126
|
-
else
|
127
|
-
new(args[0])
|
128
|
-
link_dir(args[0], "")
|
129
|
-
end
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
command :login do |c|
|
135
|
-
c.syntax = 'cnvrg login'
|
136
|
-
c.summary = 'login to cnvrg app'
|
137
|
-
c.description = 'Login to the CNVRG system'
|
138
|
-
c.option '--reset', String, 'set reset if you want to change current login credentials'
|
139
|
-
c.action do |args, options|
|
140
|
-
n = Netrc.read
|
141
|
-
user, pass = n["cnvrg.io"]
|
142
|
-
if user and pass
|
143
|
-
relogin = agree("You are already authenticated with email: #{user}, do you want to login again? (This will delete you're current login credentials")
|
144
|
-
if !relogin
|
145
|
-
exit(0)
|
146
|
-
end
|
147
|
-
end
|
148
|
-
if options.reset
|
149
|
-
n["cnvrg.io"] = "", ""
|
150
|
-
|
151
|
-
end
|
152
|
-
login_process()
|
153
|
-
|
154
|
-
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
def login_process()
|
159
|
-
n = Netrc.read
|
160
|
-
|
161
|
-
email = ask("Please enter your email")
|
162
|
-
email_regex = /\A[\w+\-.]+@[a-z\d\-]+(\.[a-z\d\-]+)*\.[a-z]+\z/i
|
163
|
-
is_valid = email_regex.match(email)
|
164
|
-
while !is_valid do
|
165
|
-
email = ask("Please enter a valid email address:")
|
166
|
-
is_valid = email_regex.match(email)
|
167
|
-
end
|
168
|
-
|
169
|
-
password = ask("Please enter your password: ") { |q| q.echo = false }
|
170
|
-
#send API to login
|
171
|
-
token = sign_in(email, password)
|
172
|
-
count = 0
|
173
|
-
while token.nil? and count<3 do
|
174
|
-
do_retry = agree("Wrong email or password, try again?")
|
175
|
-
if do_retry
|
176
|
-
count +=1
|
177
|
-
email = ask("Please enter your email")
|
178
|
-
password = ask("Please enter your password: ") { |q| q.echo = false }
|
179
|
-
token = sign_in(email, password)
|
180
|
-
end
|
181
|
-
end
|
182
|
-
if token.nil?
|
183
|
-
puts "Error: Too many sign in retries"
|
184
|
-
exit(0)
|
185
|
-
else
|
186
|
-
|
187
|
-
|
188
|
-
n.new_item_prefix = "# This entry was added automatically by CNVRG\n"
|
189
|
-
n["cnvrg.io"] = email, token
|
190
|
-
n.save
|
191
|
-
puts "Great, you're in."
|
192
|
-
return true
|
193
|
-
end
|
194
|
-
end
|
195
|
-
|
196
|
-
def sign_in(email, password)
|
197
|
-
url = URI.parse("http://localhost:3000/api/v1/users/sign_in")
|
198
|
-
|
199
|
-
req = Net::HTTP::Post.new(url.path)
|
200
|
-
req.add_field("EMAIL", email)
|
201
|
-
req.add_field("PASSWORD", password)
|
202
|
-
|
203
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
204
|
-
http.request(req)
|
205
|
-
end
|
206
|
-
|
207
|
-
result = JSON.parse(response.body)
|
208
|
-
if result["status"] == 200
|
209
|
-
return result["token"]
|
210
|
-
else
|
211
|
-
return nil
|
212
|
-
end
|
213
|
-
end
|
214
|
-
|
215
|
-
def verify_auth_exist
|
216
|
-
n = Netrc.read
|
217
|
-
user, pass = n["cnvrg.io"]
|
218
|
-
if user.nil? or pass.nil?
|
219
|
-
puts "Please login first, using \"cnvrg login\" command"
|
220
|
-
return false
|
221
|
-
end
|
222
|
-
return true
|
223
|
-
end
|
224
|
-
|
225
|
-
def get_logged_token
|
226
|
-
n = Netrc.read
|
227
|
-
user, token = n["cnvrg.io"]
|
228
|
-
return token
|
229
|
-
|
230
|
-
end
|
231
|
-
def run(is_start, body)
|
232
|
-
token = get_logged_token()
|
233
|
-
if is_start
|
234
|
-
url = URI.parse("http://localhost:3000/api/v1/cli/start_run")
|
235
|
-
else
|
236
|
-
url = URI.parse("http://localhost:3000/api/v1/cli/end_run")
|
237
|
-
end
|
238
|
-
|
239
|
-
|
240
|
-
req = Net::HTTP::Post.new(url.path)
|
241
|
-
req.add_field("AUTH_TOKEN", token)
|
242
|
-
req.body = body
|
243
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
244
|
-
http.request(req)
|
245
|
-
end
|
246
|
-
|
247
|
-
result = JSON.parse(response.body)
|
248
|
-
puts result
|
249
|
-
end
|
250
|
-
|
251
|
-
def new(project_name)
|
252
|
-
begin
|
253
|
-
`mkdir -p #{project_name} #{project_name}/data #{project_name}/models #{project_name}/notebooks #{project_name}/src #{project_name}/src/data #{project_name}/src/features #{project_name}/src/models #{project_name}/src/visualizations #{project_name}/.cnvrg`
|
254
|
-
`touch #{project_name}/README.md`
|
255
|
-
`touch #{project_name}/.cnvrgignore`
|
256
|
-
rescue
|
257
|
-
"Error"
|
258
|
-
end
|
259
|
-
end
|
260
|
-
|
261
|
-
def link_dir(project_name, user, host="cnvrg.io")
|
262
|
-
`touch #{project_name}/.cnvrg/project`
|
263
|
-
`echo "owner: #{user}\nproject_slug: #{project_name}\nhost: #{host}" >> #{project_name}/.cnvrg/project`
|
264
|
-
end
|
265
|
-
|
266
|
-
def download
|
267
|
-
|
268
|
-
status_output = status
|
269
|
-
downloadables = status_output["updated_on_server"].merge(status_output["deleted"])
|
270
|
-
|
271
|
-
|
272
|
-
downloadables.each do |d|
|
273
|
-
if d[-1] == "/"
|
274
|
-
# make dir
|
275
|
-
else
|
276
|
-
files << #download(d)
|
277
|
-
end
|
278
|
-
end
|
279
|
-
|
280
|
-
def upload
|
281
|
-
|
282
|
-
end
|
283
|
-
|
284
|
-
def generate_idx
|
285
|
-
|
286
|
-
idx = Hash.new(0)
|
287
|
-
list = Dir['**/**']
|
288
|
-
|
289
|
-
# remove files that are in CNVRGIGNORE
|
290
|
-
# list.select { |x| ! cnvrgignore.include? x }
|
291
|
-
|
292
|
-
list.each do |e|
|
293
|
-
if File.directory? e
|
294
|
-
idx[e+"/"] = nil
|
295
|
-
else
|
296
|
-
idx[e] = { sha1: Digest::SHA1.hexdigest(File.read(e)),
|
297
|
-
commit_time: nil }
|
298
|
-
end
|
299
|
-
end
|
300
|
-
|
301
|
-
File.open("hello/.cnvrg/idx.yml", 'w') { |f| f.write idx.to_yaml }
|
302
|
-
|
303
|
-
|
304
|
-
end
|
305
|
-
|
306
|
-
def status
|
307
|
-
url = URI.parse("http://localhost:3000/api/v1/users/yochze/projects/porject/status")
|
308
|
-
token = get_logged_token()
|
309
|
-
load_idx = YAML.load(File.read("hello/.cnvrg/idx.yml"))
|
310
|
-
req = Net::HTTP::Get.new(url, 'Content-Type' => 'application/json')
|
311
|
-
req.add_field("AUTH_TOKEN", token)
|
312
|
-
req.body = { snapshot: load_idx }.to_json
|
313
|
-
response = Net::HTTP.new(url.host, url.port).start do |http|
|
314
|
-
http.request(req)
|
315
|
-
end
|
316
|
-
|
317
|
-
result = JSON.parse(response.body)
|
318
|
-
|
319
|
-
result["status"]["added"].each do |a|
|
320
|
-
puts "A:\t#{a}"
|
321
|
-
end
|
322
|
-
|
323
|
-
result["status"]["updated_on_server"].each do |a|
|
324
|
-
puts "M:\t#{a}"
|
325
|
-
end
|
326
|
-
|
327
|
-
result["status"]["deleted"].each do |a|
|
328
|
-
puts "D:\t#{a}"
|
329
|
-
end
|
330
|
-
|
331
|
-
return result["status"]
|
332
|
-
end
|