cnvrg 0.0.145 → 0.0.146
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +354 -85
- data/lib/cnvrg/data.rb +26 -0
- data/lib/cnvrg/datafiles.rb +376 -0
- data/lib/cnvrg/dataset.rb +196 -0
- data/lib/cnvrg/experiment.rb +6 -4
- data/lib/cnvrg/files.rb +7 -2
- data/lib/cnvrg/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: db813c4622a353a9e9d60678849c904d4268d28f
|
4
|
+
data.tar.gz: ecbc4ec2f9acc052ba1b61b18a88f14c15eb0312
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fcaf8b6352e76e300d75ea4db289b3c58c8cfe785522baa100cafa271afea54e0987dea60ab79387e5552258f55209cb667cff2f840d61199d28b5a44c4d8e92
|
7
|
+
data.tar.gz: 42b8ba42106b5d6321c56cc94db3e434102f6323941a3084cbf8a5bbef9242f154fca542591c3fce1a83f568893f14142b61392b7848120d50c13299476f35a9
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -18,6 +18,8 @@ require 'cnvrg/project'
|
|
18
18
|
require 'cnvrg/files'
|
19
19
|
require 'cnvrg/experiment'
|
20
20
|
require 'cnvrg/Images'
|
21
|
+
require 'cnvrg/dataset'
|
22
|
+
require 'cnvrg/datafiles'
|
21
23
|
require 'etc'
|
22
24
|
require 'logstash-logger'
|
23
25
|
require 'cnvrg/job'
|
@@ -29,8 +31,12 @@ require 'fileutils'
|
|
29
31
|
require 'zip'
|
30
32
|
require 'active_support/all'
|
31
33
|
require 'thor'
|
34
|
+
require 'pathname'
|
35
|
+
require 'cnvrg/data'
|
36
|
+
|
32
37
|
|
33
38
|
# DEV VERSION
|
39
|
+
|
34
40
|
#
|
35
41
|
module Cnvrg
|
36
42
|
class CLI < Thor
|
@@ -38,18 +44,6 @@ module Cnvrg
|
|
38
44
|
INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
|
39
45
|
IP="localhost"
|
40
46
|
PORT=7654
|
41
|
-
desc '', ''
|
42
|
-
|
43
|
-
def printable_commands(all = true, subcommand = false)
|
44
|
-
(all ? all_commands : commands).map do |_, command|
|
45
|
-
next if command.hidden? or (command.description.empty? and command.usage.empty?)
|
46
|
-
item = []
|
47
|
-
item << banner(command, false, subcommand)
|
48
|
-
item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
|
49
|
-
item
|
50
|
-
end.compact
|
51
|
-
end
|
52
|
-
|
53
47
|
class << self
|
54
48
|
# Hackery.Take the run method away from Thor so that we can redefine it.
|
55
49
|
def is_thor_reserved_word?(word, type)
|
@@ -57,11 +51,11 @@ module Cnvrg
|
|
57
51
|
super
|
58
52
|
end
|
59
53
|
end
|
54
|
+
desc "data", "upload and manage datasets", :hide =>true
|
55
|
+
subcommand "data", Data
|
60
56
|
|
61
|
-
desc "", ""
|
62
|
-
|
63
|
-
|
64
|
-
def testt
|
57
|
+
desc "", "" , :hide => true
|
58
|
+
def test
|
65
59
|
# image_settings = {
|
66
60
|
# 'Image' => "cnvrg:latest",
|
67
61
|
# 'User' => 'ds',
|
@@ -86,7 +80,6 @@ module Cnvrg
|
|
86
80
|
|
87
81
|
|
88
82
|
desc 'version', 'Prints cnvrg current version'
|
89
|
-
|
90
83
|
def version
|
91
84
|
puts Cnvrg::VERSION
|
92
85
|
|
@@ -95,7 +88,6 @@ module Cnvrg
|
|
95
88
|
map %w(-v --version) => :version
|
96
89
|
|
97
90
|
desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
|
98
|
-
|
99
91
|
def set_api_url(url)
|
100
92
|
home_dir = File.expand_path('~')
|
101
93
|
if !url.end_with? "/api"
|
@@ -136,7 +128,7 @@ module Cnvrg
|
|
136
128
|
end
|
137
129
|
end
|
138
130
|
|
139
|
-
desc '', ''
|
131
|
+
desc '', '', :hide => true
|
140
132
|
|
141
133
|
def set_remote_api_url(owner, current_user, url)
|
142
134
|
home_dir = File.expand_path('~')
|
@@ -181,27 +173,41 @@ module Cnvrg
|
|
181
173
|
owners = result["owners"]
|
182
174
|
urls = result["urls"]
|
183
175
|
choose_owner = result["username"]
|
184
|
-
|
185
176
|
if owners.empty?
|
186
177
|
else
|
187
178
|
owners << choose_owner
|
188
179
|
chosen = false
|
189
180
|
while !chosen
|
190
|
-
|
181
|
+
owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
|
182
|
+
choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
|
183
|
+
|
184
|
+
if choose_owner =~ /[[:digit:]]/
|
185
|
+
ow_index = choose_owner.to_i-1
|
186
|
+
if ow_index<0 or ow_index >= owners.size
|
187
|
+
say "No such owner, please choose again", Thor::Shell::Color::BLUE
|
188
|
+
chosen = false
|
189
|
+
next
|
190
|
+
end
|
191
|
+
choose_owner = owners[choose_owner.to_i-1]
|
192
|
+
chosen = true
|
191
193
|
|
192
|
-
owners_lower = owners.map { |o| o.downcase }
|
193
|
-
ow_index = owners_lower.index(choose_owner.downcase)
|
194
|
-
if ow_index.nil?
|
195
|
-
say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
|
196
194
|
else
|
197
|
-
|
195
|
+
|
196
|
+
owners_lower = owners.map { |o| o.downcase }
|
197
|
+
ow_index = owners_lower.index(choose_owner.downcase)
|
198
|
+
if ow_index.nil?
|
199
|
+
say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
|
200
|
+
else
|
201
|
+
chosen = true
|
202
|
+
end
|
198
203
|
end
|
204
|
+
|
199
205
|
end
|
200
206
|
|
201
207
|
|
202
208
|
end
|
203
|
-
if set_owner(
|
204
|
-
say "Setting default owner: #{
|
209
|
+
if set_owner(choose_owner, result["username"], urls[ow_index])
|
210
|
+
say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
|
205
211
|
else
|
206
212
|
say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
|
207
213
|
end
|
@@ -343,8 +349,8 @@ module Cnvrg
|
|
343
349
|
|
344
350
|
## Projects
|
345
351
|
desc 'new', 'Create a new cnvrg project'
|
346
|
-
method_option :clean, :type => :boolean, :aliases => ["-c"
|
347
|
-
method_option :docker_image, :type => :string, :aliases => ["-d"
|
352
|
+
method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
|
353
|
+
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
348
354
|
|
349
355
|
def new(project_name)
|
350
356
|
begin
|
@@ -468,8 +474,8 @@ module Cnvrg
|
|
468
474
|
|
469
475
|
end
|
470
476
|
desc 'link', 'Link current directory to a new cnvrg project'
|
471
|
-
method_option :sync, :type => :boolean, :aliases => ["-s"
|
472
|
-
method_option :docker_image, :type => :string, :aliases => ["-d"
|
477
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
478
|
+
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
473
479
|
|
474
480
|
def link
|
475
481
|
begin
|
@@ -550,6 +556,252 @@ module Cnvrg
|
|
550
556
|
exit(1)
|
551
557
|
end
|
552
558
|
end
|
559
|
+
|
560
|
+
desc 'init_data', 'Init dataset directory', :hide =>true
|
561
|
+
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
562
|
+
|
563
|
+
def init_data
|
564
|
+
begin
|
565
|
+
verify_logged_in(false)
|
566
|
+
log_start(__method__, args, options)
|
567
|
+
dataset_name =File.basename(Dir.getwd)
|
568
|
+
if File.directory?(Dir.getwd+"/.cnvrg")
|
569
|
+
config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
|
570
|
+
say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
|
571
|
+
|
572
|
+
exit(0)
|
573
|
+
end
|
574
|
+
say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
|
575
|
+
|
576
|
+
working_dir = Dir.getwd
|
577
|
+
owner = CLI.get_owner
|
578
|
+
if Dataset.init(owner,dataset_name, options["public"])
|
579
|
+
path = Dir.pwd
|
580
|
+
@dataset = Dataset.new(path)
|
581
|
+
@dataset.generate_idx()
|
582
|
+
|
583
|
+
url = @dataset.url
|
584
|
+
check = Helpers.checkmark
|
585
|
+
say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
|
586
|
+
say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
|
587
|
+
log_end(0)
|
588
|
+
|
589
|
+
else
|
590
|
+
log_end(1, "can't create dataset")
|
591
|
+
@dataset.revert(working_dir) unless @dataset.nil?
|
592
|
+
say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
|
593
|
+
exit(0)
|
594
|
+
end
|
595
|
+
rescue SignalException
|
596
|
+
log_end(-1)
|
597
|
+
|
598
|
+
say "\nAborting"
|
599
|
+
exit(1)
|
600
|
+
end
|
601
|
+
end
|
602
|
+
desc 'upload_data', 'Upload data files', :hide=>true
|
603
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
604
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
|
605
|
+
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
606
|
+
method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
|
607
|
+
|
608
|
+
def upload_data(sync=false,direct=false)
|
609
|
+
|
610
|
+
begin
|
611
|
+
verify_logged_in(true)
|
612
|
+
log_start(__method__, args, options)
|
613
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
614
|
+
@dataset = Dataset.new(dataset_dir)
|
615
|
+
|
616
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
617
|
+
ignore = options[:ignore] || []
|
618
|
+
if !@dataset.update_ignore_list(ignore)
|
619
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
620
|
+
end
|
621
|
+
result = @dataset.compare_idx(false)
|
622
|
+
commit = result["result"]["commit"]
|
623
|
+
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
624
|
+
log_end(0)
|
625
|
+
|
626
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
627
|
+
exit(1)
|
628
|
+
end
|
629
|
+
|
630
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
631
|
+
result = result["result"]["tree"]
|
632
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
633
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
634
|
+
# if all.size == 1
|
635
|
+
# num = "conflict"
|
636
|
+
# else
|
637
|
+
# num = "conflicts"
|
638
|
+
# end
|
639
|
+
# say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
|
640
|
+
# say "#{all.join("\n")}"
|
641
|
+
# say "Please fix #{num}, and retry", Thor::Shell::Color::RED
|
642
|
+
# exit(1)
|
643
|
+
#
|
644
|
+
# end
|
645
|
+
check = Helpers.checkmark()
|
646
|
+
|
647
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
648
|
+
log_end(0)
|
649
|
+
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
650
|
+
return true
|
651
|
+
end
|
652
|
+
update_count = 0
|
653
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
654
|
+
successful_updates = []
|
655
|
+
successful_deletions = []
|
656
|
+
if options["verbose"]
|
657
|
+
if update_total == 1
|
658
|
+
say "Updating #{update_total} file", Thor::Shell::Color::BLUE
|
659
|
+
else
|
660
|
+
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
661
|
+
end
|
662
|
+
else
|
663
|
+
say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
|
664
|
+
|
665
|
+
end
|
666
|
+
|
667
|
+
# Start commit
|
668
|
+
|
669
|
+
commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
|
670
|
+
|
671
|
+
# upload / update
|
672
|
+
begin
|
673
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
674
|
+
absolute_path = "#{@dataset.local_path}/#{f}"
|
675
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
676
|
+
if File.directory?(absolute_path)
|
677
|
+
resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
|
678
|
+
if resDir
|
679
|
+
update_count += 1
|
680
|
+
successful_updates<< relative_path
|
681
|
+
end
|
682
|
+
else
|
683
|
+
res = @files.upload_file(absolute_path, relative_path, commit_sha1)
|
684
|
+
if res
|
685
|
+
update_count += 1
|
686
|
+
successful_updates<< relative_path
|
687
|
+
else
|
688
|
+
@files.rollback_commit(commit_sha1)
|
689
|
+
log_end(1, "can't upload, Rolling Back all changes")
|
690
|
+
say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
|
691
|
+
exit(0)
|
692
|
+
end
|
693
|
+
end
|
694
|
+
end
|
695
|
+
|
696
|
+
# delete
|
697
|
+
deleted = update_deleted(result["deleted"])
|
698
|
+
deleted.each do |f|
|
699
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
700
|
+
if relative_path.end_with?("/")
|
701
|
+
if @files.delete_dir(f, relative_path, commit_sha1)
|
702
|
+
# update_count += 1
|
703
|
+
successful_updates<< relative_path
|
704
|
+
end
|
705
|
+
else
|
706
|
+
if @files.delete_file(f, relative_path, commit_sha1)
|
707
|
+
# update_count += 1
|
708
|
+
successful_updates<< relative_path
|
709
|
+
end
|
710
|
+
end
|
711
|
+
end
|
712
|
+
|
713
|
+
rescue SignalException
|
714
|
+
log_end(-1)
|
715
|
+
@files.rollback_commit(commit_sha1)
|
716
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
717
|
+
exit(0)
|
718
|
+
rescue => e
|
719
|
+
log_end(-1, e.message)
|
720
|
+
@files.rollback_commit(commit_sha1)
|
721
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
722
|
+
exit(0)
|
723
|
+
end
|
724
|
+
if !result["deleted"].nil? and !result["deleted"].empty?
|
725
|
+
update_count += result["deleted"].size
|
726
|
+
end
|
727
|
+
if update_count == update_total
|
728
|
+
res = @files.end_commit(commit_sha1)
|
729
|
+
if (Cnvrg::CLI.is_response_success(res, false))
|
730
|
+
# save idx
|
731
|
+
begin
|
732
|
+
@dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
|
733
|
+
|
734
|
+
@dataset.update_idx_with_commit!(commit_sha1)
|
735
|
+
rescue => e
|
736
|
+
log_end(-1, e.message)
|
737
|
+
@files.rollback_commit(commit_sha1)
|
738
|
+
say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
|
739
|
+
exit(1)
|
740
|
+
|
741
|
+
end
|
742
|
+
if options["verbose"]
|
743
|
+
say "#{check} Done", Thor::Shell::Color::BLUE
|
744
|
+
if successful_updates.size >0
|
745
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
746
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
747
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
748
|
+
end
|
749
|
+
if successful_deletions.size >0
|
750
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
751
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
752
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
753
|
+
end
|
754
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
755
|
+
else
|
756
|
+
if (options["sync"] or sync) and direct
|
757
|
+
say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
|
758
|
+
|
759
|
+
else
|
760
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
761
|
+
|
762
|
+
end
|
763
|
+
|
764
|
+
end
|
765
|
+
|
766
|
+
log_end(0)
|
767
|
+
else
|
768
|
+
@files.rollback_commit(commit_sha1)
|
769
|
+
log_end(1, "error. Rolling Back all changes")
|
770
|
+
say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
|
771
|
+
end
|
772
|
+
else
|
773
|
+
log_end(1, "error. Rolling Back all changes")
|
774
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
775
|
+
|
776
|
+
@files.rollback_commit(commit_sha1)
|
777
|
+
end
|
778
|
+
rescue =>e
|
779
|
+
|
780
|
+
log_end(-1)
|
781
|
+
|
782
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
783
|
+
@files.rollback_commit(commit_sha1)
|
784
|
+
exit(1)
|
785
|
+
rescue SignalException
|
786
|
+
log_end(-1)
|
787
|
+
|
788
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
789
|
+
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
790
|
+
@files.rollback_commit(commit_sha1)
|
791
|
+
exit(1)
|
792
|
+
end
|
793
|
+
|
794
|
+
end
|
795
|
+
desc 'unlink','Unlink a project from current directory'
|
796
|
+
def create_volume
|
797
|
+
verify_logged_in(true)
|
798
|
+
log_start(__method__, args, options)
|
799
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
800
|
+
@dataset = Dataset.new(dataset_dir)
|
801
|
+
@dataset.create_volume()
|
802
|
+
|
803
|
+
end
|
804
|
+
|
553
805
|
desc 'unlink','Unlink a project from current directory'
|
554
806
|
def unlink
|
555
807
|
verify_logged_in(false)
|
@@ -733,7 +985,7 @@ module Cnvrg
|
|
733
985
|
exit(1)
|
734
986
|
end
|
735
987
|
end
|
736
|
-
desc '', '
|
988
|
+
desc '', '' , :hide => true
|
737
989
|
def revert_exp
|
738
990
|
begin
|
739
991
|
log_start(__method__, args, options)
|
@@ -754,12 +1006,12 @@ module Cnvrg
|
|
754
1006
|
|
755
1007
|
|
756
1008
|
desc 'upload', 'Upload updated files'
|
757
|
-
method_option :ignore, :type => :array, :aliases => ["-i"
|
758
|
-
method_option :new_branch, :type => :boolean, :aliases => ["-nb"
|
759
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
760
|
-
method_option :sync, :type => :boolean, :aliases => ["
|
1009
|
+
method_option :ignore, :type => :array, :aliases => ["-i"], :desc => "ignore following files"
|
1010
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1011
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1012
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
761
1013
|
|
762
|
-
def upload(link=false, sync=false)
|
1014
|
+
def upload(link=false, sync=false,direct=false)
|
763
1015
|
|
764
1016
|
begin
|
765
1017
|
verify_logged_in(true)
|
@@ -807,9 +1059,11 @@ module Cnvrg
|
|
807
1059
|
# exit(1)
|
808
1060
|
#
|
809
1061
|
# end
|
1062
|
+
check = Helpers.checkmark()
|
1063
|
+
|
810
1064
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
811
1065
|
log_end(0)
|
812
|
-
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
1066
|
+
say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
|
813
1067
|
return true
|
814
1068
|
end
|
815
1069
|
update_count = 0
|
@@ -906,7 +1160,6 @@ module Cnvrg
|
|
906
1160
|
if image and image.is_docker
|
907
1161
|
image.update_image_activity(commit_sha1, nil)
|
908
1162
|
end
|
909
|
-
check = Helpers.checkmark()
|
910
1163
|
|
911
1164
|
if options["verbose"]
|
912
1165
|
say "#{check} Done", Thor::Shell::Color::BLUE
|
@@ -922,10 +1175,10 @@ module Cnvrg
|
|
922
1175
|
end
|
923
1176
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
924
1177
|
else
|
925
|
-
if
|
926
|
-
|
1178
|
+
if (options["sync"] or sync) and direct
|
1179
|
+
say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
927
1180
|
|
928
|
-
|
1181
|
+
else
|
929
1182
|
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
930
1183
|
|
931
1184
|
end
|
@@ -940,7 +1193,7 @@ module Cnvrg
|
|
940
1193
|
end
|
941
1194
|
else
|
942
1195
|
log_end(1, "error. Rolling Back all changes")
|
943
|
-
say "Error occurd, \nRolling back", Thor::Shell::Color::
|
1196
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::RED
|
944
1197
|
|
945
1198
|
@files.rollback_commit(commit_sha1)
|
946
1199
|
end
|
@@ -948,7 +1201,7 @@ module Cnvrg
|
|
948
1201
|
|
949
1202
|
log_end(-1)
|
950
1203
|
|
951
|
-
say "Error occurd, \nAborting", Thor::Shell::Color::
|
1204
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
952
1205
|
@files.rollback_commit(commit_sha1)
|
953
1206
|
exit(1)
|
954
1207
|
rescue SignalException
|
@@ -963,9 +1216,9 @@ module Cnvrg
|
|
963
1216
|
end
|
964
1217
|
|
965
1218
|
desc 'download', 'Download updated files'
|
966
|
-
method_option :new_branch, :type => :boolean, :aliases => ["-nb"
|
967
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
968
|
-
method_option :sync, :type => :boolean, :aliases => ["
|
1219
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1220
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1221
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
969
1222
|
|
970
1223
|
def download(sync=false)
|
971
1224
|
begin
|
@@ -1050,7 +1303,7 @@ module Cnvrg
|
|
1050
1303
|
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
1051
1304
|
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
1052
1305
|
else
|
1053
|
-
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
1306
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
|
1054
1307
|
end
|
1055
1308
|
|
1056
1309
|
|
@@ -1163,10 +1416,11 @@ module Cnvrg
|
|
1163
1416
|
|
1164
1417
|
|
1165
1418
|
desc 'sync', 'Sync with remote server'
|
1166
|
-
method_option :new_branch, :type => :boolean, :aliases => ["
|
1167
|
-
method_option :verbose, :type => :boolean, :aliases => ["
|
1419
|
+
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1420
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1168
1421
|
|
1169
|
-
def sync
|
1422
|
+
def sync(direct=true)
|
1423
|
+
verify_logged_in(true) if direct
|
1170
1424
|
if options["verbose"]
|
1171
1425
|
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
1172
1426
|
else
|
@@ -1175,7 +1429,7 @@ module Cnvrg
|
|
1175
1429
|
|
1176
1430
|
|
1177
1431
|
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
|
1178
|
-
invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
|
1432
|
+
invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
|
1179
1433
|
|
1180
1434
|
|
1181
1435
|
end
|
@@ -1227,7 +1481,7 @@ module Cnvrg
|
|
1227
1481
|
end
|
1228
1482
|
|
1229
1483
|
|
1230
|
-
desc '', ''
|
1484
|
+
desc '', '' , :hide => true
|
1231
1485
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
|
1232
1486
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
|
1233
1487
|
method_option :title, :type => :string, :aliases => ["-t"], :default => ""
|
@@ -1244,7 +1498,8 @@ module Cnvrg
|
|
1244
1498
|
memory_average = 0
|
1245
1499
|
verify_logged_in(true)
|
1246
1500
|
log_start(__method__, args, options)
|
1247
|
-
working_dir =
|
1501
|
+
working_dir = is_cnvrg_dir
|
1502
|
+
script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
|
1248
1503
|
|
1249
1504
|
|
1250
1505
|
sync_before = options["sync_before"]
|
@@ -1268,7 +1523,7 @@ module Cnvrg
|
|
1268
1523
|
else
|
1269
1524
|
if sync_before
|
1270
1525
|
# Sync before run
|
1271
|
-
invoke :sync, [], :new_branch => is_new_branch
|
1526
|
+
invoke :sync, [false], :new_branch => is_new_branch
|
1272
1527
|
end
|
1273
1528
|
end
|
1274
1529
|
#set image for the project
|
@@ -1308,8 +1563,8 @@ module Cnvrg
|
|
1308
1563
|
platform = RUBY_PLATFORM
|
1309
1564
|
machine_name = Socket.gethostname
|
1310
1565
|
begin
|
1311
|
-
machine_activity = @exp.get_machine_activity(
|
1312
|
-
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
|
1566
|
+
machine_activity = @exp.get_machine_activity(working_dir)
|
1567
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
|
1313
1568
|
unless @exp.slug.nil?
|
1314
1569
|
real = Time.now
|
1315
1570
|
exp_success = true
|
@@ -1418,8 +1673,9 @@ module Cnvrg
|
|
1418
1673
|
if sync_after
|
1419
1674
|
# Sync after run
|
1420
1675
|
|
1676
|
+
|
1421
1677
|
download(sync=true)
|
1422
|
-
upload(link=false, sync=true)
|
1678
|
+
upload(link=false, sync=true,direct=false)
|
1423
1679
|
|
1424
1680
|
end
|
1425
1681
|
end_commit = @project.last_local_commit
|
@@ -1457,7 +1713,7 @@ module Cnvrg
|
|
1457
1713
|
end
|
1458
1714
|
end
|
1459
1715
|
|
1460
|
-
desc '', ''
|
1716
|
+
desc '', '', :hide => true
|
1461
1717
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
|
1462
1718
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
|
1463
1719
|
method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
|
@@ -1642,7 +1898,7 @@ module Cnvrg
|
|
1642
1898
|
end
|
1643
1899
|
end
|
1644
1900
|
|
1645
|
-
desc '', ''
|
1901
|
+
desc '', '', :hide => true
|
1646
1902
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
|
1647
1903
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
|
1648
1904
|
method_option :title, :type => :string, :aliases => ["-t"], :default => ""
|
@@ -1659,6 +1915,8 @@ module Cnvrg
|
|
1659
1915
|
verify_logged_in(true)
|
1660
1916
|
log_start(__method__, args, options)
|
1661
1917
|
working_dir = is_cnvrg_dir
|
1918
|
+
path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
|
1919
|
+
|
1662
1920
|
begin
|
1663
1921
|
grid = options["grid"] || nil
|
1664
1922
|
|
@@ -1726,12 +1984,12 @@ module Cnvrg
|
|
1726
1984
|
|
1727
1985
|
|
1728
1986
|
|
1729
|
-
invoke :sync, [], []
|
1987
|
+
invoke :sync, [false], []
|
1730
1988
|
|
1731
1989
|
say "Running remote experiment", Thor::Shell::Color::BLUE
|
1732
1990
|
exp = Experiment.new(project.owner, project.slug)
|
1733
1991
|
|
1734
|
-
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid)
|
1992
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
|
1735
1993
|
if Cnvrg::CLI.is_response_success(res)
|
1736
1994
|
|
1737
1995
|
# if res["result"]["machine"] == -1
|
@@ -1831,7 +2089,7 @@ module Cnvrg
|
|
1831
2089
|
|
1832
2090
|
|
1833
2091
|
|
1834
|
-
invoke :sync, [], []
|
2092
|
+
invoke :sync, [false], []
|
1835
2093
|
|
1836
2094
|
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
1837
2095
|
|
@@ -1898,9 +2156,8 @@ module Cnvrg
|
|
1898
2156
|
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
1899
2157
|
method_option :image, :type => :string, :aliases => ["--i"], :default => ""
|
1900
2158
|
|
1901
|
-
desc '
|
1902
|
-
|
1903
|
-
def notebook()
|
2159
|
+
desc 'notebook', 'starts a notebook session remotely or locally'
|
2160
|
+
def notebook
|
1904
2161
|
local = options["local"]
|
1905
2162
|
notebook_dir = options["notebook_dir"]
|
1906
2163
|
kernel = options["kernel"]
|
@@ -1961,7 +2218,7 @@ module Cnvrg
|
|
1961
2218
|
# end
|
1962
2219
|
end
|
1963
2220
|
|
1964
|
-
invoke :sync, [], []
|
2221
|
+
invoke :sync, [false], []
|
1965
2222
|
|
1966
2223
|
|
1967
2224
|
|
@@ -2004,7 +2261,7 @@ module Cnvrg
|
|
2004
2261
|
note_url = res["result"]["notebook_url"]
|
2005
2262
|
@image.set_note_url(note_url)
|
2006
2263
|
check = Helpers.checkmark()
|
2007
|
-
say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@
|
2264
|
+
say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
|
2008
2265
|
# Launchy.open(url)
|
2009
2266
|
|
2010
2267
|
exit(0)
|
@@ -2209,7 +2466,7 @@ module Cnvrg
|
|
2209
2466
|
end
|
2210
2467
|
|
2211
2468
|
end
|
2212
|
-
invoke :sync, [], :verbose => options["verbose"]
|
2469
|
+
invoke :sync, [false], :verbose => options["verbose"]
|
2213
2470
|
say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
|
2214
2471
|
#replace url
|
2215
2472
|
base_url = get_base_url()
|
@@ -2288,7 +2545,6 @@ module Cnvrg
|
|
2288
2545
|
method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
2289
2546
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
|
2290
2547
|
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
2291
|
-
|
2292
2548
|
def notebook_stop
|
2293
2549
|
begin
|
2294
2550
|
verify_logged_in(true)
|
@@ -2299,7 +2555,7 @@ module Cnvrg
|
|
2299
2555
|
|
2300
2556
|
|
2301
2557
|
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
|
2302
|
-
invoke :sync, [], :verbose => options["verbose"]
|
2558
|
+
invoke :sync, [false], :verbose => options["verbose"]
|
2303
2559
|
|
2304
2560
|
say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
|
2305
2561
|
|
@@ -2472,7 +2728,7 @@ module Cnvrg
|
|
2472
2728
|
end
|
2473
2729
|
|
2474
2730
|
end
|
2475
|
-
desc '
|
2731
|
+
desc 'build', 'run commands inside containers'
|
2476
2732
|
method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
|
2477
2733
|
|
2478
2734
|
def build(*cmd)
|
@@ -2627,7 +2883,7 @@ module Cnvrg
|
|
2627
2883
|
|
2628
2884
|
end
|
2629
2885
|
|
2630
|
-
desc 'push
|
2886
|
+
desc 'push', 'push image to cnvrg repository'
|
2631
2887
|
|
2632
2888
|
def push(*name)
|
2633
2889
|
verify_logged_in(true)
|
@@ -2734,7 +2990,7 @@ module Cnvrg
|
|
2734
2990
|
end
|
2735
2991
|
end
|
2736
2992
|
|
2737
|
-
desc '', ''
|
2993
|
+
desc '', '', :hide => true
|
2738
2994
|
|
2739
2995
|
def upload_log()
|
2740
2996
|
log_path = '/home/ds/app/uwsgi.log'
|
@@ -2743,7 +2999,7 @@ module Cnvrg
|
|
2743
2999
|
|
2744
3000
|
end
|
2745
3001
|
|
2746
|
-
desc '', ''
|
3002
|
+
desc '', '', :hide => true
|
2747
3003
|
|
2748
3004
|
def exec_container(container_id, *cmd)
|
2749
3005
|
container = Docker::Container.get(container_id)
|
@@ -2755,7 +3011,7 @@ module Cnvrg
|
|
2755
3011
|
|
2756
3012
|
end
|
2757
3013
|
|
2758
|
-
desc '', ''
|
3014
|
+
desc '', '', :hide => true
|
2759
3015
|
|
2760
3016
|
def port_container(container_id)
|
2761
3017
|
container = Docker::Container.get(container_id)
|
@@ -2764,7 +3020,7 @@ module Cnvrg
|
|
2764
3020
|
|
2765
3021
|
end
|
2766
3022
|
|
2767
|
-
desc '', ''
|
3023
|
+
desc '', '', :hide => true
|
2768
3024
|
|
2769
3025
|
def stop_container(container_id)
|
2770
3026
|
container = Docker::Container.get(container_id)
|
@@ -2773,7 +3029,7 @@ module Cnvrg
|
|
2773
3029
|
|
2774
3030
|
end
|
2775
3031
|
|
2776
|
-
desc '', ''
|
3032
|
+
desc '', '', :hide => true
|
2777
3033
|
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
2778
3034
|
method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
|
2779
3035
|
method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
@@ -2836,7 +3092,7 @@ module Cnvrg
|
|
2836
3092
|
return false
|
2837
3093
|
end
|
2838
3094
|
end
|
2839
|
-
desc '', ''
|
3095
|
+
desc '', '', :hide => true
|
2840
3096
|
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
2841
3097
|
method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
|
2842
3098
|
method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
@@ -2904,7 +3160,7 @@ module Cnvrg
|
|
2904
3160
|
end
|
2905
3161
|
end
|
2906
3162
|
|
2907
|
-
desc '', ''
|
3163
|
+
desc '', '', :hide => true
|
2908
3164
|
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
2909
3165
|
|
2910
3166
|
def config_flask_remote(image_name, port=80)
|
@@ -2959,7 +3215,7 @@ module Cnvrg
|
|
2959
3215
|
end
|
2960
3216
|
end
|
2961
3217
|
|
2962
|
-
desc '', ''
|
3218
|
+
desc '', '', :hide => true
|
2963
3219
|
|
2964
3220
|
def upload_cnvrg_image(image_name)
|
2965
3221
|
verify_logged_in(false)
|
@@ -3020,7 +3276,7 @@ module Cnvrg
|
|
3020
3276
|
end
|
3021
3277
|
end
|
3022
3278
|
|
3023
|
-
desc '', ''
|
3279
|
+
desc '', '', :hide => true
|
3024
3280
|
|
3025
3281
|
def download_image(image_name, image_slug)
|
3026
3282
|
begin
|
@@ -3397,7 +3653,6 @@ module Cnvrg
|
|
3397
3653
|
def self.is_response_success(response, should_exit=true)
|
3398
3654
|
if response.nil?
|
3399
3655
|
if !Cnvrg::Helpers.internet_connection?
|
3400
|
-
# Cnvrg::CLI.log_end(1,"no internet connection")
|
3401
3656
|
say("<%= color('Error:You seems to be offline', RED) %>")
|
3402
3657
|
else
|
3403
3658
|
say("<%= color('Error', RED) %>")
|
@@ -3543,11 +3798,15 @@ module Cnvrg
|
|
3543
3798
|
def is_cnvrg_dir(dir=Dir.pwd)
|
3544
3799
|
current_dir = dir
|
3545
3800
|
home_dir = File.expand_path('~')
|
3801
|
+
if current_dir.eql? home_dir
|
3802
|
+
return false
|
3803
|
+
end
|
3546
3804
|
is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
|
3547
3805
|
until is_cnvrg == true
|
3548
3806
|
current_dir = File.expand_path("..", current_dir)
|
3549
3807
|
is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
|
3550
|
-
if File.expand_path("..", current_dir).eql? home_dir
|
3808
|
+
if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
|
3809
|
+
is_cnvrg = false
|
3551
3810
|
break
|
3552
3811
|
end
|
3553
3812
|
end
|
@@ -3753,6 +4012,16 @@ module Cnvrg
|
|
3753
4012
|
|
3754
4013
|
return final
|
3755
4014
|
end
|
4015
|
+
def get_cmd_path_in_dir(main_dir,sub_dir)
|
4016
|
+
first = Pathname.new main_dir
|
4017
|
+
second = Pathname.new sub_dir
|
4018
|
+
relative = second.relative_path_from first
|
4019
|
+
if relative.eql? "."
|
4020
|
+
return ""
|
4021
|
+
else
|
4022
|
+
return relative
|
4023
|
+
end
|
4024
|
+
end
|
3756
4025
|
|
3757
4026
|
|
3758
4027
|
|