cnvrg 0.0.3 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/cnvrg +6 -2
- data/cnvrg.gemspec +7 -0
- data/lib/cnvrg.rb +0 -1
- data/lib/cnvrg/api.rb +3 -1
- data/lib/cnvrg/auth.rb +2 -2
- data/lib/cnvrg/cli.rb +215 -67
- data/lib/cnvrg/experiment.rb +12 -3
- data/lib/cnvrg/files.rb +7 -5
- data/lib/cnvrg/project.rb +3 -2
- data/lib/cnvrg/runner.rb +49 -0
- data/lib/cnvrg/version.rb +2 -1
- metadata +74 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7754ddbb96eae3b876dbf3714d6d11a044d65081
|
4
|
+
data.tar.gz: 68d1487897427b1fc27fb87c6965a64f68921ca1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 014f26db1691f330c27ed186db235e3c8b90e41ea316c9fa6df7f3fb18eed5e17f567713fdbec3dc30824621d748a9ad729a487b968c515b513e7d7b557ef3a9
|
7
|
+
data.tar.gz: 0db14fdb44be449fa6d9924a8aa8454f78e03e7f38157f636b5d8a6799917a672d32f3ac0bf10574e71c7da910c0b1f10d1e128901683a47f9a32351d7cb465f
|
data/bin/cnvrg
CHANGED
data/cnvrg.gemspec
CHANGED
@@ -17,6 +17,13 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
18
|
spec.executables = ['cnvrg']
|
19
19
|
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_development_dependency 'bundler', '~> 1.11'
|
22
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
23
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
24
|
+
spec.add_development_dependency 'vcr', '~> 3.0'
|
25
|
+
spec.add_development_dependency 'aruba'
|
26
|
+
|
20
27
|
spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
|
21
28
|
spec.add_runtime_dependency 'faraday', '~> 0.10.0'
|
22
29
|
spec.add_runtime_dependency 'netrc', '~> 0.11.0'
|
data/lib/cnvrg.rb
CHANGED
data/lib/cnvrg/api.rb
CHANGED
@@ -2,13 +2,15 @@ require 'netrc'
|
|
2
2
|
require 'faraday'
|
3
3
|
require 'json'
|
4
4
|
require 'fileutils'
|
5
|
+
require 'cnvrg/helpers'
|
5
6
|
|
6
7
|
module Cnvrg
|
7
8
|
class API
|
8
9
|
USER_AGENT = "CnvrgCLI/#{Cnvrg::VERSION}"
|
9
|
-
#
|
10
|
+
#ENDPOINT = 'http://localhost:3000/api'
|
10
11
|
ENDPOINT = 'https://cnvrg.io/api'
|
11
12
|
ENDPOINT_VERSION = 'v1'
|
13
|
+
URL = "#{ENDPOINT}/#{ENDPOINT_VERSION}"
|
12
14
|
|
13
15
|
def self.request(resource, method = 'GET', data = {}, parse_request = true)
|
14
16
|
begin
|
data/lib/cnvrg/auth.rb
CHANGED
@@ -45,7 +45,7 @@ module Cnvrg
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def sign_in(email, password)
|
48
|
-
url = API
|
48
|
+
url = API::URL
|
49
49
|
url = URI.parse(url+ "/users/sign_in")
|
50
50
|
http = Net::HTTP.new(url.host, url.port)
|
51
51
|
|
@@ -61,7 +61,7 @@ module Cnvrg
|
|
61
61
|
|
62
62
|
result = JSON.parse(response.body)
|
63
63
|
if result["status"] == 200
|
64
|
-
return result["
|
64
|
+
return result["result"]
|
65
65
|
else
|
66
66
|
return nil
|
67
67
|
end
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -21,14 +21,37 @@ require 'cnvrg/experiment'
|
|
21
21
|
#
|
22
22
|
module Cnvrg
|
23
23
|
class CLI < Thor
|
24
|
-
map %w[--version -v] => :__print_version
|
25
24
|
|
26
|
-
desc
|
25
|
+
desc 'version', 'Prints cnvrg current version'
|
27
26
|
|
28
|
-
def
|
27
|
+
def version
|
29
28
|
puts Cnvrg::VERSION
|
30
29
|
end
|
31
30
|
|
31
|
+
map %w(-v --version) => :version
|
32
|
+
desc 'set default owner', 'set default owner'
|
33
|
+
|
34
|
+
def set_default_owner
|
35
|
+
config = YAML.load_file("~/.cnvrg/config.yml")
|
36
|
+
username = config.to_h[:username]
|
37
|
+
res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
|
38
|
+
if Cnvrg::CLI.is_response_success(res)
|
39
|
+
owner = username
|
40
|
+
result = res["result"]
|
41
|
+
if result["owners"].size > 1
|
42
|
+
owner = ask("Choose default owner:\n"+result["owners"].join("\n")+"\n")
|
43
|
+
|
44
|
+
end
|
45
|
+
if set_owner(owner, username)
|
46
|
+
say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
|
47
|
+
else
|
48
|
+
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
map %w(--set-default-owner) => :set_default_owner
|
54
|
+
|
32
55
|
|
33
56
|
desc 'login', 'Authenticate with cnvrg.io and store credentials'
|
34
57
|
|
@@ -39,7 +62,8 @@ module Cnvrg
|
|
39
62
|
|
40
63
|
@auth = Cnvrg::Auth.new
|
41
64
|
netrc = Netrc.read
|
42
|
-
@email, token = netrc[Cnvrg::Helpers.netrc_domain]
|
65
|
+
@email, token, owner = netrc[Cnvrg::Helpers.netrc_domain]
|
66
|
+
puts owner
|
43
67
|
|
44
68
|
if @email and token
|
45
69
|
say 'Seems you\'re already logged in', Thor::Shell::Color::BLUE
|
@@ -47,12 +71,26 @@ module Cnvrg
|
|
47
71
|
end
|
48
72
|
@email = ask("Enter your Email:")
|
49
73
|
password = cmd.ask("Enter your password (hidden):") { |q| q.echo = "*" }
|
50
|
-
|
51
|
-
if (token =
|
74
|
+
result = @auth.sign_in(@email, password)
|
75
|
+
if (token = result["token"])
|
52
76
|
netrc[Cnvrg::Helpers.netrc_domain] = @email, token
|
53
77
|
netrc.save
|
54
78
|
|
55
79
|
say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
|
80
|
+
owners = result["owners"]
|
81
|
+
choose_owner = result["username"]
|
82
|
+
|
83
|
+
if owners.empty?
|
84
|
+
else
|
85
|
+
owners << choose_owner
|
86
|
+
choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
|
87
|
+
|
88
|
+
end
|
89
|
+
if set_owner(choose_owner, result["username"])
|
90
|
+
say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
|
91
|
+
else
|
92
|
+
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
93
|
+
end
|
56
94
|
|
57
95
|
else
|
58
96
|
say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
|
@@ -291,7 +329,7 @@ module Cnvrg
|
|
291
329
|
successful_updates<< relative_path
|
292
330
|
end
|
293
331
|
else
|
294
|
-
|
332
|
+
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
333
|
if res
|
296
334
|
update_count += 1
|
297
335
|
successful_updates<< relative_path
|
@@ -323,32 +361,36 @@ module Cnvrg
|
|
323
361
|
@files.rollback_commit(commit_sha1)
|
324
362
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
325
363
|
exit(0)
|
364
|
+
rescue
|
365
|
+
@files.rollback_commit(commit_sha1)
|
366
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
367
|
+
exit(0)
|
326
368
|
end
|
327
369
|
if update_count == update_total
|
328
370
|
res = @files.end_commit(commit_sha1)
|
329
371
|
if (Cnvrg::CLI.is_response_success(res, false))
|
330
372
|
# save idx
|
331
373
|
begin
|
332
|
-
|
374
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
333
375
|
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
376
|
+
@project.update_idx_with_commit!(commit_sha1)
|
377
|
+
rescue
|
378
|
+
@files.rollback_commit(commit_sha1)
|
379
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
380
|
+
exit(1)
|
339
381
|
|
340
|
-
|
382
|
+
end
|
341
383
|
|
342
|
-
|
384
|
+
say "Done", Thor::Shell::Color::BLUE
|
343
385
|
if successful_updates.size >0
|
344
386
|
say "Updated:", Thor::Shell::Color::GREEN
|
345
|
-
|
346
|
-
|
387
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
388
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
347
389
|
end
|
348
390
|
if successful_deletions.size >0
|
349
391
|
say "Deleted:", Thor::Shell::Color::GREEN
|
350
|
-
|
351
|
-
|
392
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
393
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
352
394
|
end
|
353
395
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
354
396
|
else
|
@@ -427,6 +469,27 @@ module Cnvrg
|
|
427
469
|
invoke :upload
|
428
470
|
end
|
429
471
|
|
472
|
+
# desc 'random', 'random'
|
473
|
+
# def random
|
474
|
+
# say "Fun trivia game for taking a recess :-)", Thor::Shell::Color::BLUE
|
475
|
+
# subject = ask("Pick a subject\n1.NBA\n2.American History\n3.Data Science\n")
|
476
|
+
# file = "ds.txt"
|
477
|
+
# case subject
|
478
|
+
# when "1"
|
479
|
+
# file = "nba.txt"
|
480
|
+
# when "2"
|
481
|
+
# file = "ah.txt"
|
482
|
+
# when "3"
|
483
|
+
# file = "ds.txt"
|
484
|
+
# end
|
485
|
+
# line = File.readlines(file).sample
|
486
|
+
# q = line[0,line.index('?')+1]
|
487
|
+
# a = line[line.index('?')+1,line.size]
|
488
|
+
# answer = ask(q+"/n")
|
489
|
+
#
|
490
|
+
#
|
491
|
+
#
|
492
|
+
# end
|
430
493
|
# Run
|
431
494
|
#
|
432
495
|
desc 'exec CMD', 'Execute a process'
|
@@ -434,6 +497,8 @@ module Cnvrg
|
|
434
497
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
435
498
|
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
436
499
|
method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
|
500
|
+
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
|
501
|
+
method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
|
437
502
|
|
438
503
|
def exec(*cmd)
|
439
504
|
verify_logged_in()
|
@@ -444,6 +509,10 @@ module Cnvrg
|
|
444
509
|
sync_after = options["sync_after"]
|
445
510
|
print_log = options["log"]
|
446
511
|
title = options["title"]
|
512
|
+
email_notification = options["email_notification"]
|
513
|
+
upload_output = options["upload_output"]
|
514
|
+
time_to_upload = calc_output_time(upload_output)
|
515
|
+
|
447
516
|
if sync_before
|
448
517
|
# Sync before run
|
449
518
|
say "Syncing project before running", Thor::Shell::Color::BLUE
|
@@ -462,19 +531,21 @@ module Cnvrg
|
|
462
531
|
|
463
532
|
platform = RUBY_PLATFORM
|
464
533
|
machine_name = Socket.gethostname
|
534
|
+
begin
|
535
|
+
|
536
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification)
|
537
|
+
unless @exp.slug.nil?
|
538
|
+
real = Time.now
|
539
|
+
exp_success = true
|
540
|
+
memory_total = []
|
541
|
+
cpu_total = []
|
542
|
+
start_loop = Time.now
|
465
543
|
|
466
|
-
@exp.start(cmd, platform, machine_name, start_commit, title)
|
467
|
-
unless @exp.slug.nil?
|
468
|
-
real = Time.now
|
469
|
-
cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
|
470
|
-
exp_success = true
|
471
|
-
memory_total = []
|
472
|
-
cpu_total = []
|
473
|
-
begin
|
474
544
|
|
475
545
|
PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
|
476
546
|
begin
|
477
547
|
stdout.each do |line|
|
548
|
+
|
478
549
|
cur_time = Time.now
|
479
550
|
monitor = %x{ps aux|awk '{print $2,$3,$4}'|grep #{pid} }
|
480
551
|
monitor_by = monitor.split(" ")
|
@@ -482,17 +553,35 @@ module Cnvrg
|
|
482
553
|
cpu = monitor_by[1]
|
483
554
|
memory_total << memory.to_f
|
484
555
|
cpu_total << cpu.to_f
|
556
|
+
real_time= Time.now-real
|
557
|
+
|
485
558
|
cur_log = {time: cur_time,
|
486
559
|
message: line,
|
487
560
|
type: "stdout",
|
488
561
|
rss: memory,
|
489
562
|
cpu: cpu,
|
490
|
-
real:
|
563
|
+
real: real_time}
|
491
564
|
if print_log
|
492
565
|
puts cur_log
|
493
566
|
end
|
494
|
-
|
495
567
|
log << cur_log
|
568
|
+
begin
|
569
|
+
if time_to_upload !=0
|
570
|
+
if time_to_upload <= Time.now - start_loop
|
571
|
+
#upload current log
|
572
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
573
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
574
|
+
|
575
|
+
@exp.upload_temp_log(log, cpu_average, memory_average)
|
576
|
+
log = []
|
577
|
+
start_loop = Time.now
|
578
|
+
end
|
579
|
+
|
580
|
+
end
|
581
|
+
rescue
|
582
|
+
say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
|
583
|
+
end
|
584
|
+
|
496
585
|
|
497
586
|
end
|
498
587
|
if stderr
|
@@ -503,56 +592,115 @@ module Cnvrg
|
|
503
592
|
Process.wait(pid)
|
504
593
|
rescue Errno::EIO
|
505
594
|
break
|
595
|
+
rescue Errno::ENOENT
|
596
|
+
exp_success = false
|
597
|
+
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
598
|
+
rescue PTY::ChildExited
|
599
|
+
exp_success = false
|
600
|
+
puts "The process exited!"
|
601
|
+
end
|
602
|
+
|
603
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
604
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
605
|
+
exit_status = $?.exitstatus
|
606
|
+
if !exp_success
|
607
|
+
end_commit = @project.last_local_commit
|
608
|
+
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
609
|
+
say "Experiment has failed", Thor::Shell::Color::RED
|
610
|
+
exit(0)
|
506
611
|
end
|
612
|
+
if sync_after
|
613
|
+
say "Syncing project after running", Thor::Shell::Color::BLUE
|
614
|
+
# Sync after run
|
615
|
+
download()
|
616
|
+
upload()
|
617
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
618
|
+
end
|
619
|
+
end_commit = @project.last_local_commit
|
620
|
+
|
621
|
+
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
622
|
+
check = Helpers.checkmark()
|
623
|
+
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
507
624
|
end
|
508
|
-
|
509
|
-
exp_success = false
|
510
|
-
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
511
|
-
rescue PTY::ChildExited
|
512
|
-
exp_success = false
|
513
|
-
puts "The process exited!"
|
514
|
-
end
|
515
|
-
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
516
|
-
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
517
|
-
exit_status = $?.exitstatus
|
518
|
-
if !exp_success
|
519
|
-
end_commit = @project.last_local_commit
|
520
|
-
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
521
|
-
say "Experiment has failed", Thor::Shell::Color::RED
|
522
|
-
exit(0)
|
523
|
-
end
|
524
|
-
if sync_after
|
525
|
-
say "Syncing project after running", Thor::Shell::Color::BLUE
|
526
|
-
# Sync after run
|
527
|
-
download()
|
528
|
-
upload()
|
529
|
-
say "Done Syncing", Thor::Shell::Color::BLUE
|
625
|
+
|
530
626
|
end
|
531
|
-
end_commit = @project.last_local_commit
|
532
627
|
|
533
|
-
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
534
|
-
check = Helpers.checkmark()
|
535
|
-
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
536
|
-
else
|
537
|
-
# didnt run
|
538
628
|
end
|
539
629
|
end
|
540
630
|
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
631
|
+
|
632
|
+
no_tasks do
|
633
|
+
def set_owner(owner, username)
|
634
|
+
begin
|
635
|
+
if !File.directory? "~/.cnvrg"
|
636
|
+
FileUtils.mkdir_p("~/.cnvrg")
|
637
|
+
end
|
638
|
+
if !File.exist?("~/.cnvrg/config.yml")
|
639
|
+
FileUtils.touch ["~/.cnvrg/config.yml"]
|
640
|
+
end
|
641
|
+
config = {owner: owner, username: username}
|
642
|
+
File.open("~/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
643
|
+
return true
|
644
|
+
rescue
|
548
645
|
return false
|
549
646
|
end
|
647
|
+
|
550
648
|
end
|
551
|
-
return true
|
552
|
-
end
|
553
649
|
|
554
|
-
|
650
|
+
def calc_output_time(upload_output)
|
651
|
+
puts upload_output
|
652
|
+
if upload_output.nil? or upload_output.empty?
|
653
|
+
return 0
|
654
|
+
end
|
655
|
+
time = upload_output.split(/(\d+)/).reject(&:empty?).map { |x| x.strip }
|
656
|
+
if time.size!=2
|
657
|
+
upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\nre-enter value for upload_output")
|
658
|
+
return calc_output_time(upload_output)
|
659
|
+
end
|
660
|
+
case time[1].downcase
|
661
|
+
when "s"
|
662
|
+
return time[0].to_f
|
663
|
+
when "m"
|
664
|
+
return time[0].to_f*60
|
665
|
+
when "h"
|
666
|
+
return time[0].to_f*3600
|
667
|
+
when "d"
|
668
|
+
return time[0].to_f*24*3600
|
669
|
+
else
|
670
|
+
upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\n re-enter value for upload_output")
|
671
|
+
calc_output_time(upload_output)
|
672
|
+
end
|
673
|
+
|
674
|
+
end
|
555
675
|
|
676
|
+
def self.is_response_success(response, should_exit=true)
|
677
|
+
if response["status"]!= 200
|
678
|
+
error = response['message']
|
679
|
+
if response["status"] == 500
|
680
|
+
say("<%= color('Server Error', RED) %>")
|
681
|
+
else
|
682
|
+
say("<%= color('Error: #{error}', RED) %>")
|
683
|
+
end
|
684
|
+
|
685
|
+
if should_exit
|
686
|
+
exit(1)
|
687
|
+
else
|
688
|
+
return false
|
689
|
+
end
|
690
|
+
end
|
691
|
+
return true
|
692
|
+
end
|
693
|
+
|
694
|
+
def self.get_owner
|
695
|
+
config = YAML.load_file("~/.cnvrg/config.yml")
|
696
|
+
owner = config.to_h[:owner]
|
697
|
+
if owner.empty?
|
698
|
+
invoke :set_default_owner
|
699
|
+
return get_owner()
|
700
|
+
else
|
701
|
+
return owner
|
702
|
+
end
|
703
|
+
end
|
556
704
|
|
557
705
|
def get_project_home
|
558
706
|
absolute_path = Dir.pwd
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -8,8 +8,9 @@ module Cnvrg
|
|
8
8
|
@slug = nil
|
9
9
|
end
|
10
10
|
|
11
|
-
def start(input, platform, machine_name, start_commit,name)
|
12
|
-
res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
|
11
|
+
def start(input, platform, machine_name, start_commit,name, email_notification)
|
12
|
+
res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
|
13
|
+
{ input: input, platform: platform, machine_name: machine_name, start_commit: start_commit , title:name, email_notification:email_notification})
|
13
14
|
Cnvrg::CLI.is_response_success(res)
|
14
15
|
|
15
16
|
@slug = res.to_h["result"].to_h["slug"]
|
@@ -18,8 +19,16 @@ module Cnvrg
|
|
18
19
|
|
19
20
|
end
|
20
21
|
|
22
|
+
def upload_temp_log(temp_log, cpu_average, memory_average )
|
23
|
+
response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', { output: temp_log,
|
24
|
+
exp_slug: @slug,cpu_average:cpu_average,
|
25
|
+
memory_average:memory_average})
|
26
|
+
Cnvrg::CLI.is_response_success(response)
|
27
|
+
end
|
21
28
|
def end(output, exit_status, end_commit,cpu_average, memory_average)
|
22
|
-
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug,
|
29
|
+
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug,
|
30
|
+
exit_status: exit_status, end_commit: end_commit,
|
31
|
+
cpu_average:cpu_average,memory_average:memory_average })
|
23
32
|
Cnvrg::CLI.is_response_success(response)
|
24
33
|
end
|
25
34
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -13,11 +13,14 @@ module Cnvrg
|
|
13
13
|
def upload_file(absolute_path, relative_path, commit_sha1)
|
14
14
|
file_name = File.basename relative_path
|
15
15
|
file_size = File.size(relative_path).to_f
|
16
|
+
mime_type = MimeMagic.by_path(relative_path)
|
17
|
+
content_type = !mime_type.nil? ? mime_type.type : ""
|
16
18
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', { absolute_path: absolute_path, relative_path: relative_path,
|
17
|
-
commit_sha1: commit_sha1, file_name: file_name ,
|
19
|
+
commit_sha1: commit_sha1, file_name: file_name ,
|
20
|
+
file_size:file_size,file_content_type:content_type})
|
18
21
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
19
22
|
path = upload_resp["result"]["path"]
|
20
|
-
s3_res = upload_s3(path,relative_path)
|
23
|
+
s3_res = upload_s3(path,relative_path,content_type)
|
21
24
|
if s3_res
|
22
25
|
Cnvrg::API.request(@base_resource + "update_s3", 'POST', { path:path,commit_id:upload_resp["result"]["commit_id"],
|
23
26
|
blob_id:upload_resp["result"]["blob_id"]})
|
@@ -26,10 +29,9 @@ module Cnvrg
|
|
26
29
|
end
|
27
30
|
return false
|
28
31
|
end
|
29
|
-
def upload_s3(url,file)
|
32
|
+
def upload_s3(url,file,content_type)
|
30
33
|
url = URI.parse(url)
|
31
|
-
|
32
|
-
content_type = !mime_type.nil? ? mime_type.type : ""
|
34
|
+
|
33
35
|
file = File.open(file, "rb")
|
34
36
|
body = file.read
|
35
37
|
begin
|
data/lib/cnvrg/project.rb
CHANGED
@@ -86,11 +86,12 @@ module Cnvrg
|
|
86
86
|
cnvrgignore = Helpers.cnvrgignore_content
|
87
87
|
|
88
88
|
begin
|
89
|
-
|
89
|
+
|
90
|
+
owner = Cnvrg::CLI.get_owner()
|
91
|
+
response = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name, owner:owner })
|
90
92
|
Cnvrg::CLI.is_response_success(response)
|
91
93
|
response = JSON.parse response["result"]
|
92
94
|
project_slug = response["slug"]
|
93
|
-
owner = response["owner"]
|
94
95
|
|
95
96
|
config = { project_name: project_name,
|
96
97
|
project_slug: project_slug,
|
data/lib/cnvrg/runner.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'cnvrg'
|
2
|
+
|
3
|
+
module Cnvrg
|
4
|
+
class Runner
|
5
|
+
# Allow everything fun to be injected from the outside while defaulting to normal implementations.
|
6
|
+
def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
|
7
|
+
@argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute!
|
11
|
+
exit_code = begin
|
12
|
+
# Thor accesses these streams directly rather than letting them be injected, so we replace them...
|
13
|
+
$stderr = @stderr
|
14
|
+
$stdin = @stdin
|
15
|
+
$stdout = @stdout
|
16
|
+
|
17
|
+
# Run our normal Thor app the way we know and love.
|
18
|
+
Cnvrg::CLI.start(@argv)
|
19
|
+
|
20
|
+
# Thor::Base#start does not have a return value, assume success if no exception is raised.
|
21
|
+
0
|
22
|
+
rescue StandardError => e
|
23
|
+
# The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception
|
24
|
+
b = e.backtrace
|
25
|
+
@stderr.puts("#{b.shift}: #{e.message} (#{e.class})")
|
26
|
+
@stderr.puts(b.map{|s| "\tfrom #{s}"}.join("\n"))
|
27
|
+
1
|
28
|
+
rescue SystemExit => e
|
29
|
+
e.status
|
30
|
+
ensure
|
31
|
+
# TODO: reset your app here, free up resources, etc.
|
32
|
+
# Examples:
|
33
|
+
# MyApp.logger.flush
|
34
|
+
# MyApp.logger.close
|
35
|
+
# MyApp.logger = nil
|
36
|
+
#
|
37
|
+
# MyApp.reset_singleton_instance_variables
|
38
|
+
|
39
|
+
# ...then we put the streams back.
|
40
|
+
$stderr = STDERR
|
41
|
+
$stdin = STDIN
|
42
|
+
$stdout = STDOUT
|
43
|
+
end
|
44
|
+
|
45
|
+
# Proxy our exit code back to the injected kernel.
|
46
|
+
@kernel.exit(exit_code)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,8 +9,78 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-12-
|
12
|
+
date: 2016-12-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.11'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.11'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '10.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '10.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rspec
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '3.0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '3.0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: vcr
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '3.0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: aruba
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
14
84
|
- !ruby/object:Gem::Dependency
|
15
85
|
name: mimemagic
|
16
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -137,6 +207,7 @@ files:
|
|
137
207
|
- lib/cnvrg/files.rb
|
138
208
|
- lib/cnvrg/helpers.rb
|
139
209
|
- lib/cnvrg/project.rb
|
210
|
+
- lib/cnvrg/runner.rb
|
140
211
|
- lib/cnvrg/version.rb
|
141
212
|
homepage: https://cnvrg.io
|
142
213
|
licenses: []
|
@@ -157,7 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
157
228
|
version: '0'
|
158
229
|
requirements: []
|
159
230
|
rubyforge_project:
|
160
|
-
rubygems_version: 2.5.
|
231
|
+
rubygems_version: 2.5.2
|
161
232
|
signing_key:
|
162
233
|
specification_version: 4
|
163
234
|
summary: A CLI tool for interacting with cnvrg.io.
|