cnvrg 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/cnvrg +6 -2
- data/cnvrg.gemspec +7 -0
- data/lib/cnvrg.rb +0 -1
- data/lib/cnvrg/api.rb +3 -1
- data/lib/cnvrg/auth.rb +2 -2
- data/lib/cnvrg/cli.rb +215 -67
- data/lib/cnvrg/experiment.rb +12 -3
- data/lib/cnvrg/files.rb +7 -5
- data/lib/cnvrg/project.rb +3 -2
- data/lib/cnvrg/runner.rb +49 -0
- data/lib/cnvrg/version.rb +2 -1
- metadata +74 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7754ddbb96eae3b876dbf3714d6d11a044d65081
|
4
|
+
data.tar.gz: 68d1487897427b1fc27fb87c6965a64f68921ca1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 014f26db1691f330c27ed186db235e3c8b90e41ea316c9fa6df7f3fb18eed5e17f567713fdbec3dc30824621d748a9ad729a487b968c515b513e7d7b557ef3a9
|
7
|
+
data.tar.gz: 0db14fdb44be449fa6d9924a8aa8454f78e03e7f38157f636b5d8a6799917a672d32f3ac0bf10574e71c7da910c0b1f10d1e128901683a47f9a32351d7cb465f
|
data/bin/cnvrg
CHANGED
data/cnvrg.gemspec
CHANGED
@@ -17,6 +17,13 @@ Gem::Specification.new do |spec|
|
|
17
17
|
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
18
18
|
spec.executables = ['cnvrg']
|
19
19
|
spec.require_paths = ['lib']
|
20
|
+
|
21
|
+
spec.add_development_dependency 'bundler', '~> 1.11'
|
22
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
23
|
+
spec.add_development_dependency 'rspec', '~> 3.0'
|
24
|
+
spec.add_development_dependency 'vcr', '~> 3.0'
|
25
|
+
spec.add_development_dependency 'aruba'
|
26
|
+
|
20
27
|
spec.add_runtime_dependency 'mimemagic', '~> 0.3.1','>=0.3.2'
|
21
28
|
spec.add_runtime_dependency 'faraday', '~> 0.10.0'
|
22
29
|
spec.add_runtime_dependency 'netrc', '~> 0.11.0'
|
data/lib/cnvrg.rb
CHANGED
data/lib/cnvrg/api.rb
CHANGED
@@ -2,13 +2,15 @@ require 'netrc'
|
|
2
2
|
require 'faraday'
|
3
3
|
require 'json'
|
4
4
|
require 'fileutils'
|
5
|
+
require 'cnvrg/helpers'
|
5
6
|
|
6
7
|
module Cnvrg
|
7
8
|
class API
|
8
9
|
USER_AGENT = "CnvrgCLI/#{Cnvrg::VERSION}"
|
9
|
-
#
|
10
|
+
#ENDPOINT = 'http://localhost:3000/api'
|
10
11
|
ENDPOINT = 'https://cnvrg.io/api'
|
11
12
|
ENDPOINT_VERSION = 'v1'
|
13
|
+
URL = "#{ENDPOINT}/#{ENDPOINT_VERSION}"
|
12
14
|
|
13
15
|
def self.request(resource, method = 'GET', data = {}, parse_request = true)
|
14
16
|
begin
|
data/lib/cnvrg/auth.rb
CHANGED
@@ -45,7 +45,7 @@ module Cnvrg
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def sign_in(email, password)
|
48
|
-
url = API
|
48
|
+
url = API::URL
|
49
49
|
url = URI.parse(url+ "/users/sign_in")
|
50
50
|
http = Net::HTTP.new(url.host, url.port)
|
51
51
|
|
@@ -61,7 +61,7 @@ module Cnvrg
|
|
61
61
|
|
62
62
|
result = JSON.parse(response.body)
|
63
63
|
if result["status"] == 200
|
64
|
-
return result["
|
64
|
+
return result["result"]
|
65
65
|
else
|
66
66
|
return nil
|
67
67
|
end
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -21,14 +21,37 @@ require 'cnvrg/experiment'
|
|
21
21
|
#
|
22
22
|
module Cnvrg
|
23
23
|
class CLI < Thor
|
24
|
-
map %w[--version -v] => :__print_version
|
25
24
|
|
26
|
-
desc
|
25
|
+
desc 'version', 'Prints cnvrg current version'
|
27
26
|
|
28
|
-
def
|
27
|
+
def version
|
29
28
|
puts Cnvrg::VERSION
|
30
29
|
end
|
31
30
|
|
31
|
+
map %w(-v --version) => :version
|
32
|
+
desc 'set default owner', 'set default owner'
|
33
|
+
|
34
|
+
def set_default_owner
|
35
|
+
config = YAML.load_file("~/.cnvrg/config.yml")
|
36
|
+
username = config.to_h[:username]
|
37
|
+
res = Cnvrg::API.request("/users/#{username}/get_possible_owners", 'GET')
|
38
|
+
if Cnvrg::CLI.is_response_success(res)
|
39
|
+
owner = username
|
40
|
+
result = res["result"]
|
41
|
+
if result["owners"].size > 1
|
42
|
+
owner = ask("Choose default owner:\n"+result["owners"].join("\n")+"\n")
|
43
|
+
|
44
|
+
end
|
45
|
+
if set_owner(owner, username)
|
46
|
+
say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
|
47
|
+
else
|
48
|
+
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
map %w(--set-default-owner) => :set_default_owner
|
54
|
+
|
32
55
|
|
33
56
|
desc 'login', 'Authenticate with cnvrg.io and store credentials'
|
34
57
|
|
@@ -39,7 +62,8 @@ module Cnvrg
|
|
39
62
|
|
40
63
|
@auth = Cnvrg::Auth.new
|
41
64
|
netrc = Netrc.read
|
42
|
-
@email, token = netrc[Cnvrg::Helpers.netrc_domain]
|
65
|
+
@email, token, owner = netrc[Cnvrg::Helpers.netrc_domain]
|
66
|
+
puts owner
|
43
67
|
|
44
68
|
if @email and token
|
45
69
|
say 'Seems you\'re already logged in', Thor::Shell::Color::BLUE
|
@@ -47,12 +71,26 @@ module Cnvrg
|
|
47
71
|
end
|
48
72
|
@email = ask("Enter your Email:")
|
49
73
|
password = cmd.ask("Enter your password (hidden):") { |q| q.echo = "*" }
|
50
|
-
|
51
|
-
if (token =
|
74
|
+
result = @auth.sign_in(@email, password)
|
75
|
+
if (token = result["token"])
|
52
76
|
netrc[Cnvrg::Helpers.netrc_domain] = @email, token
|
53
77
|
netrc.save
|
54
78
|
|
55
79
|
say "Authenticated successfully as #{@email}", Thor::Shell::Color::GREEN
|
80
|
+
owners = result["owners"]
|
81
|
+
choose_owner = result["username"]
|
82
|
+
|
83
|
+
if owners.empty?
|
84
|
+
else
|
85
|
+
owners << choose_owner
|
86
|
+
choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
|
87
|
+
|
88
|
+
end
|
89
|
+
if set_owner(choose_owner, result["username"])
|
90
|
+
say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
|
91
|
+
else
|
92
|
+
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
93
|
+
end
|
56
94
|
|
57
95
|
else
|
58
96
|
say "Failed to authenticate, wrong email/password", Thor::Shell::Color::RED
|
@@ -291,7 +329,7 @@ module Cnvrg
|
|
291
329
|
successful_updates<< relative_path
|
292
330
|
end
|
293
331
|
else
|
294
|
-
|
332
|
+
res = @files.upload_file(f, relative_path, commit_sha1)
|
295
333
|
if res
|
296
334
|
update_count += 1
|
297
335
|
successful_updates<< relative_path
|
@@ -323,32 +361,36 @@ module Cnvrg
|
|
323
361
|
@files.rollback_commit(commit_sha1)
|
324
362
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
325
363
|
exit(0)
|
364
|
+
rescue
|
365
|
+
@files.rollback_commit(commit_sha1)
|
366
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
367
|
+
exit(0)
|
326
368
|
end
|
327
369
|
if update_count == update_total
|
328
370
|
res = @files.end_commit(commit_sha1)
|
329
371
|
if (Cnvrg::CLI.is_response_success(res, false))
|
330
372
|
# save idx
|
331
373
|
begin
|
332
|
-
|
374
|
+
@project.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
333
375
|
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
|
338
|
-
|
376
|
+
@project.update_idx_with_commit!(commit_sha1)
|
377
|
+
rescue
|
378
|
+
@files.rollback_commit(commit_sha1)
|
379
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
380
|
+
exit(1)
|
339
381
|
|
340
|
-
|
382
|
+
end
|
341
383
|
|
342
|
-
|
384
|
+
say "Done", Thor::Shell::Color::BLUE
|
343
385
|
if successful_updates.size >0
|
344
386
|
say "Updated:", Thor::Shell::Color::GREEN
|
345
|
-
|
346
|
-
|
387
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
388
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
347
389
|
end
|
348
390
|
if successful_deletions.size >0
|
349
391
|
say "Deleted:", Thor::Shell::Color::GREEN
|
350
|
-
|
351
|
-
|
392
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
393
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
352
394
|
end
|
353
395
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
354
396
|
else
|
@@ -427,6 +469,27 @@ module Cnvrg
|
|
427
469
|
invoke :upload
|
428
470
|
end
|
429
471
|
|
472
|
+
# desc 'random', 'random'
|
473
|
+
# def random
|
474
|
+
# say "Fun trivia game for taking a recess :-)", Thor::Shell::Color::BLUE
|
475
|
+
# subject = ask("Pick a subject\n1.NBA\n2.American History\n3.Data Science\n")
|
476
|
+
# file = "ds.txt"
|
477
|
+
# case subject
|
478
|
+
# when "1"
|
479
|
+
# file = "nba.txt"
|
480
|
+
# when "2"
|
481
|
+
# file = "ah.txt"
|
482
|
+
# when "3"
|
483
|
+
# file = "ds.txt"
|
484
|
+
# end
|
485
|
+
# line = File.readlines(file).sample
|
486
|
+
# q = line[0,line.index('?')+1]
|
487
|
+
# a = line[line.index('?')+1,line.size]
|
488
|
+
# answer = ask(q+"/n")
|
489
|
+
#
|
490
|
+
#
|
491
|
+
#
|
492
|
+
# end
|
430
493
|
# Run
|
431
494
|
#
|
432
495
|
desc 'exec CMD', 'Execute a process'
|
@@ -434,6 +497,8 @@ module Cnvrg
|
|
434
497
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
435
498
|
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
436
499
|
method_option :log, :type => :boolean, :aliases => ["-l", "--l"], :default => false
|
500
|
+
method_option :email_notification, :type => :boolean, :aliases => ["-en", "--en"], :default => false
|
501
|
+
method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
|
437
502
|
|
438
503
|
def exec(*cmd)
|
439
504
|
verify_logged_in()
|
@@ -444,6 +509,10 @@ module Cnvrg
|
|
444
509
|
sync_after = options["sync_after"]
|
445
510
|
print_log = options["log"]
|
446
511
|
title = options["title"]
|
512
|
+
email_notification = options["email_notification"]
|
513
|
+
upload_output = options["upload_output"]
|
514
|
+
time_to_upload = calc_output_time(upload_output)
|
515
|
+
|
447
516
|
if sync_before
|
448
517
|
# Sync before run
|
449
518
|
say "Syncing project before running", Thor::Shell::Color::BLUE
|
@@ -462,19 +531,21 @@ module Cnvrg
|
|
462
531
|
|
463
532
|
platform = RUBY_PLATFORM
|
464
533
|
machine_name = Socket.gethostname
|
534
|
+
begin
|
535
|
+
|
536
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification)
|
537
|
+
unless @exp.slug.nil?
|
538
|
+
real = Time.now
|
539
|
+
exp_success = true
|
540
|
+
memory_total = []
|
541
|
+
cpu_total = []
|
542
|
+
start_loop = Time.now
|
465
543
|
|
466
|
-
@exp.start(cmd, platform, machine_name, start_commit, title)
|
467
|
-
unless @exp.slug.nil?
|
468
|
-
real = Time.now
|
469
|
-
cpu = Process.clock_gettime(Process::CLOCK_PROCESS_CPUTIME_ID)
|
470
|
-
exp_success = true
|
471
|
-
memory_total = []
|
472
|
-
cpu_total = []
|
473
|
-
begin
|
474
544
|
|
475
545
|
PTY.spawn(cmd) do |stdout, stdin, pid, stderr|
|
476
546
|
begin
|
477
547
|
stdout.each do |line|
|
548
|
+
|
478
549
|
cur_time = Time.now
|
479
550
|
monitor = %x{ps aux|awk '{print $2,$3,$4}'|grep #{pid} }
|
480
551
|
monitor_by = monitor.split(" ")
|
@@ -482,17 +553,35 @@ module Cnvrg
|
|
482
553
|
cpu = monitor_by[1]
|
483
554
|
memory_total << memory.to_f
|
484
555
|
cpu_total << cpu.to_f
|
556
|
+
real_time= Time.now-real
|
557
|
+
|
485
558
|
cur_log = {time: cur_time,
|
486
559
|
message: line,
|
487
560
|
type: "stdout",
|
488
561
|
rss: memory,
|
489
562
|
cpu: cpu,
|
490
|
-
real:
|
563
|
+
real: real_time}
|
491
564
|
if print_log
|
492
565
|
puts cur_log
|
493
566
|
end
|
494
|
-
|
495
567
|
log << cur_log
|
568
|
+
begin
|
569
|
+
if time_to_upload !=0
|
570
|
+
if time_to_upload <= Time.now - start_loop
|
571
|
+
#upload current log
|
572
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
573
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
574
|
+
|
575
|
+
@exp.upload_temp_log(log, cpu_average, memory_average)
|
576
|
+
log = []
|
577
|
+
start_loop = Time.now
|
578
|
+
end
|
579
|
+
|
580
|
+
end
|
581
|
+
rescue
|
582
|
+
say "Failed to upload ongoing results, continuing with experiment", Thor::Shell::Color::YELLOW
|
583
|
+
end
|
584
|
+
|
496
585
|
|
497
586
|
end
|
498
587
|
if stderr
|
@@ -503,56 +592,115 @@ module Cnvrg
|
|
503
592
|
Process.wait(pid)
|
504
593
|
rescue Errno::EIO
|
505
594
|
break
|
595
|
+
rescue Errno::ENOENT
|
596
|
+
exp_success = false
|
597
|
+
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
598
|
+
rescue PTY::ChildExited
|
599
|
+
exp_success = false
|
600
|
+
puts "The process exited!"
|
601
|
+
end
|
602
|
+
|
603
|
+
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
604
|
+
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
605
|
+
exit_status = $?.exitstatus
|
606
|
+
if !exp_success
|
607
|
+
end_commit = @project.last_local_commit
|
608
|
+
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
609
|
+
say "Experiment has failed", Thor::Shell::Color::RED
|
610
|
+
exit(0)
|
506
611
|
end
|
612
|
+
if sync_after
|
613
|
+
say "Syncing project after running", Thor::Shell::Color::BLUE
|
614
|
+
# Sync after run
|
615
|
+
download()
|
616
|
+
upload()
|
617
|
+
say "Done Syncing", Thor::Shell::Color::BLUE
|
618
|
+
end
|
619
|
+
end_commit = @project.last_local_commit
|
620
|
+
|
621
|
+
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
622
|
+
check = Helpers.checkmark()
|
623
|
+
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
507
624
|
end
|
508
|
-
|
509
|
-
exp_success = false
|
510
|
-
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
511
|
-
rescue PTY::ChildExited
|
512
|
-
exp_success = false
|
513
|
-
puts "The process exited!"
|
514
|
-
end
|
515
|
-
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
516
|
-
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
517
|
-
exit_status = $?.exitstatus
|
518
|
-
if !exp_success
|
519
|
-
end_commit = @project.last_local_commit
|
520
|
-
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
521
|
-
say "Experiment has failed", Thor::Shell::Color::RED
|
522
|
-
exit(0)
|
523
|
-
end
|
524
|
-
if sync_after
|
525
|
-
say "Syncing project after running", Thor::Shell::Color::BLUE
|
526
|
-
# Sync after run
|
527
|
-
download()
|
528
|
-
upload()
|
529
|
-
say "Done Syncing", Thor::Shell::Color::BLUE
|
625
|
+
|
530
626
|
end
|
531
|
-
end_commit = @project.last_local_commit
|
532
627
|
|
533
|
-
res = @exp.end(log, exit_status, end_commit,cpu_average,memory_average)
|
534
|
-
check = Helpers.checkmark()
|
535
|
-
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
536
|
-
else
|
537
|
-
# didnt run
|
538
628
|
end
|
539
629
|
end
|
540
630
|
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
|
547
|
-
|
631
|
+
|
632
|
+
no_tasks do
|
633
|
+
def set_owner(owner, username)
|
634
|
+
begin
|
635
|
+
if !File.directory? "~/.cnvrg"
|
636
|
+
FileUtils.mkdir_p("~/.cnvrg")
|
637
|
+
end
|
638
|
+
if !File.exist?("~/.cnvrg/config.yml")
|
639
|
+
FileUtils.touch ["~/.cnvrg/config.yml"]
|
640
|
+
end
|
641
|
+
config = {owner: owner, username: username}
|
642
|
+
File.open("~/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
643
|
+
return true
|
644
|
+
rescue
|
548
645
|
return false
|
549
646
|
end
|
647
|
+
|
550
648
|
end
|
551
|
-
return true
|
552
|
-
end
|
553
649
|
|
554
|
-
|
650
|
+
def calc_output_time(upload_output)
|
651
|
+
puts upload_output
|
652
|
+
if upload_output.nil? or upload_output.empty?
|
653
|
+
return 0
|
654
|
+
end
|
655
|
+
time = upload_output.split(/(\d+)/).reject(&:empty?).map { |x| x.strip }
|
656
|
+
if time.size!=2
|
657
|
+
upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\nre-enter value for upload_output")
|
658
|
+
return calc_output_time(upload_output)
|
659
|
+
end
|
660
|
+
case time[1].downcase
|
661
|
+
when "s"
|
662
|
+
return time[0].to_f
|
663
|
+
when "m"
|
664
|
+
return time[0].to_f*60
|
665
|
+
when "h"
|
666
|
+
return time[0].to_f*3600
|
667
|
+
when "d"
|
668
|
+
return time[0].to_f*24*3600
|
669
|
+
else
|
670
|
+
upload_output = ask("upload_output should be {number}{s/m/h/d} i.e. 5m (5 minutes), 1h (1 hour)\n re-enter value for upload_output")
|
671
|
+
calc_output_time(upload_output)
|
672
|
+
end
|
673
|
+
|
674
|
+
end
|
555
675
|
|
676
|
+
def self.is_response_success(response, should_exit=true)
|
677
|
+
if response["status"]!= 200
|
678
|
+
error = response['message']
|
679
|
+
if response["status"] == 500
|
680
|
+
say("<%= color('Server Error', RED) %>")
|
681
|
+
else
|
682
|
+
say("<%= color('Error: #{error}', RED) %>")
|
683
|
+
end
|
684
|
+
|
685
|
+
if should_exit
|
686
|
+
exit(1)
|
687
|
+
else
|
688
|
+
return false
|
689
|
+
end
|
690
|
+
end
|
691
|
+
return true
|
692
|
+
end
|
693
|
+
|
694
|
+
def self.get_owner
|
695
|
+
config = YAML.load_file("~/.cnvrg/config.yml")
|
696
|
+
owner = config.to_h[:owner]
|
697
|
+
if owner.empty?
|
698
|
+
invoke :set_default_owner
|
699
|
+
return get_owner()
|
700
|
+
else
|
701
|
+
return owner
|
702
|
+
end
|
703
|
+
end
|
556
704
|
|
557
705
|
def get_project_home
|
558
706
|
absolute_path = Dir.pwd
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -8,8 +8,9 @@ module Cnvrg
|
|
8
8
|
@slug = nil
|
9
9
|
end
|
10
10
|
|
11
|
-
def start(input, platform, machine_name, start_commit,name)
|
12
|
-
res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
|
11
|
+
def start(input, platform, machine_name, start_commit,name, email_notification)
|
12
|
+
res = Cnvrg::API.request(@base_resource + "experiment/start", 'POST',
|
13
|
+
{ input: input, platform: platform, machine_name: machine_name, start_commit: start_commit , title:name, email_notification:email_notification})
|
13
14
|
Cnvrg::CLI.is_response_success(res)
|
14
15
|
|
15
16
|
@slug = res.to_h["result"].to_h["slug"]
|
@@ -18,8 +19,16 @@ module Cnvrg
|
|
18
19
|
|
19
20
|
end
|
20
21
|
|
22
|
+
def upload_temp_log(temp_log, cpu_average, memory_average )
|
23
|
+
response = Cnvrg::API.request(@base_resource + "experiment/upload_temp_log", 'POST', { output: temp_log,
|
24
|
+
exp_slug: @slug,cpu_average:cpu_average,
|
25
|
+
memory_average:memory_average})
|
26
|
+
Cnvrg::CLI.is_response_success(response)
|
27
|
+
end
|
21
28
|
def end(output, exit_status, end_commit,cpu_average, memory_average)
|
22
|
-
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug,
|
29
|
+
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', { output: output, exp_slug: @slug,
|
30
|
+
exit_status: exit_status, end_commit: end_commit,
|
31
|
+
cpu_average:cpu_average,memory_average:memory_average })
|
23
32
|
Cnvrg::CLI.is_response_success(response)
|
24
33
|
end
|
25
34
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -13,11 +13,14 @@ module Cnvrg
|
|
13
13
|
def upload_file(absolute_path, relative_path, commit_sha1)
|
14
14
|
file_name = File.basename relative_path
|
15
15
|
file_size = File.size(relative_path).to_f
|
16
|
+
mime_type = MimeMagic.by_path(relative_path)
|
17
|
+
content_type = !mime_type.nil? ? mime_type.type : ""
|
16
18
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', { absolute_path: absolute_path, relative_path: relative_path,
|
17
|
-
commit_sha1: commit_sha1, file_name: file_name ,
|
19
|
+
commit_sha1: commit_sha1, file_name: file_name ,
|
20
|
+
file_size:file_size,file_content_type:content_type})
|
18
21
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
19
22
|
path = upload_resp["result"]["path"]
|
20
|
-
s3_res = upload_s3(path,relative_path)
|
23
|
+
s3_res = upload_s3(path,relative_path,content_type)
|
21
24
|
if s3_res
|
22
25
|
Cnvrg::API.request(@base_resource + "update_s3", 'POST', { path:path,commit_id:upload_resp["result"]["commit_id"],
|
23
26
|
blob_id:upload_resp["result"]["blob_id"]})
|
@@ -26,10 +29,9 @@ module Cnvrg
|
|
26
29
|
end
|
27
30
|
return false
|
28
31
|
end
|
29
|
-
def upload_s3(url,file)
|
32
|
+
def upload_s3(url,file,content_type)
|
30
33
|
url = URI.parse(url)
|
31
|
-
|
32
|
-
content_type = !mime_type.nil? ? mime_type.type : ""
|
34
|
+
|
33
35
|
file = File.open(file, "rb")
|
34
36
|
body = file.read
|
35
37
|
begin
|
data/lib/cnvrg/project.rb
CHANGED
@@ -86,11 +86,12 @@ module Cnvrg
|
|
86
86
|
cnvrgignore = Helpers.cnvrgignore_content
|
87
87
|
|
88
88
|
begin
|
89
|
-
|
89
|
+
|
90
|
+
owner = Cnvrg::CLI.get_owner()
|
91
|
+
response = Cnvrg::API.request("cli/create_project", 'POST', { title: project_name, owner:owner })
|
90
92
|
Cnvrg::CLI.is_response_success(response)
|
91
93
|
response = JSON.parse response["result"]
|
92
94
|
project_slug = response["slug"]
|
93
|
-
owner = response["owner"]
|
94
95
|
|
95
96
|
config = { project_name: project_name,
|
96
97
|
project_slug: project_slug,
|
data/lib/cnvrg/runner.rb
ADDED
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'cnvrg'
|
2
|
+
|
3
|
+
module Cnvrg
|
4
|
+
class Runner
|
5
|
+
# Allow everything fun to be injected from the outside while defaulting to normal implementations.
|
6
|
+
def initialize(argv, stdin = STDIN, stdout = STDOUT, stderr = STDERR, kernel = Kernel)
|
7
|
+
@argv, @stdin, @stdout, @stderr, @kernel = argv, stdin, stdout, stderr, kernel
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute!
|
11
|
+
exit_code = begin
|
12
|
+
# Thor accesses these streams directly rather than letting them be injected, so we replace them...
|
13
|
+
$stderr = @stderr
|
14
|
+
$stdin = @stdin
|
15
|
+
$stdout = @stdout
|
16
|
+
|
17
|
+
# Run our normal Thor app the way we know and love.
|
18
|
+
Cnvrg::CLI.start(@argv)
|
19
|
+
|
20
|
+
# Thor::Base#start does not have a return value, assume success if no exception is raised.
|
21
|
+
0
|
22
|
+
rescue StandardError => e
|
23
|
+
# The ruby interpreter would pipe this to STDERR and exit 1 in the case of an unhandled exception
|
24
|
+
b = e.backtrace
|
25
|
+
@stderr.puts("#{b.shift}: #{e.message} (#{e.class})")
|
26
|
+
@stderr.puts(b.map{|s| "\tfrom #{s}"}.join("\n"))
|
27
|
+
1
|
28
|
+
rescue SystemExit => e
|
29
|
+
e.status
|
30
|
+
ensure
|
31
|
+
# TODO: reset your app here, free up resources, etc.
|
32
|
+
# Examples:
|
33
|
+
# MyApp.logger.flush
|
34
|
+
# MyApp.logger.close
|
35
|
+
# MyApp.logger = nil
|
36
|
+
#
|
37
|
+
# MyApp.reset_singleton_instance_variables
|
38
|
+
|
39
|
+
# ...then we put the streams back.
|
40
|
+
$stderr = STDERR
|
41
|
+
$stdin = STDIN
|
42
|
+
$stdout = STDOUT
|
43
|
+
end
|
44
|
+
|
45
|
+
# Proxy our exit code back to the injected kernel.
|
46
|
+
@kernel.exit(exit_code)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,8 +9,78 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-12-
|
12
|
+
date: 2016-12-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.11'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.11'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '10.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '10.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rspec
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '3.0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '3.0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: vcr
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - "~>"
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '3.0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - "~>"
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: aruba
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :development
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
14
84
|
- !ruby/object:Gem::Dependency
|
15
85
|
name: mimemagic
|
16
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -137,6 +207,7 @@ files:
|
|
137
207
|
- lib/cnvrg/files.rb
|
138
208
|
- lib/cnvrg/helpers.rb
|
139
209
|
- lib/cnvrg/project.rb
|
210
|
+
- lib/cnvrg/runner.rb
|
140
211
|
- lib/cnvrg/version.rb
|
141
212
|
homepage: https://cnvrg.io
|
142
213
|
licenses: []
|
@@ -157,7 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
157
228
|
version: '0'
|
158
229
|
requirements: []
|
159
230
|
rubyforge_project:
|
160
|
-
rubygems_version: 2.5.
|
231
|
+
rubygems_version: 2.5.2
|
161
232
|
signing_key:
|
162
233
|
specification_version: 4
|
163
234
|
summary: A CLI tool for interacting with cnvrg.io.
|