cnvrg 0.0.148 → 0.0.149
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +681 -161
- data/lib/cnvrg/data.rb +50 -6
- data/lib/cnvrg/datafiles.rb +143 -17
- data/lib/cnvrg/dataset.rb +97 -16
- data/lib/cnvrg/experiment.rb +3 -3
- data/lib/cnvrg/files.rb +11 -5
- data/lib/cnvrg/project.rb +17 -8
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 89c404e15ae2c15390c06632e16fdb46725a7dab
|
4
|
+
data.tar.gz: 28435401db7d5d2a78ba88de8946513ffc86dca5
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5442ff1a286b258d5624ba1bc00d7944423a99b0b74128cd1b90dffd24511e52e9609a7119ba2d24c4bec348927f7a637fb669b7f961d780f6070103e8cb8e20
|
7
|
+
data.tar.gz: 678610c32a2104cab60cec8bcab410cc5d76372079b99044776e444cc7713f206cb10b87521425459e3a8c53937032665a795fd36adfa48a58c9227a1963b998
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -33,11 +33,13 @@ require 'active_support/all'
|
|
33
33
|
require 'thor'
|
34
34
|
require 'pathname'
|
35
35
|
require 'cnvrg/data'
|
36
|
-
|
37
36
|
# DEV VERSION
|
38
37
|
|
39
|
-
#
|
38
|
+
# include Cnvrg::Tar
|
39
|
+
|
40
|
+
# include Tar
|
40
41
|
module Cnvrg
|
42
|
+
|
41
43
|
class CLI < Thor
|
42
44
|
|
43
45
|
INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
|
@@ -50,10 +52,11 @@ module Cnvrg
|
|
50
52
|
super
|
51
53
|
end
|
52
54
|
end
|
53
|
-
desc "data", "upload and manage datasets", :hide =>true
|
55
|
+
desc "data", "upload and manage datasets", :hide => true
|
54
56
|
subcommand "data", Data
|
55
57
|
|
56
|
-
desc "", ""
|
58
|
+
desc "", "", :hide => true
|
59
|
+
|
57
60
|
def test
|
58
61
|
# image_settings = {
|
59
62
|
# 'Image' => "cnvrg:latest",
|
@@ -74,11 +77,11 @@ module Cnvrg
|
|
74
77
|
# s = "/leah/1/2/3/4/5"
|
75
78
|
# command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
76
79
|
# puts container.exec(command, tty: true)
|
77
|
-
puts File.read("s4cmd.py")
|
78
80
|
end
|
79
81
|
|
80
82
|
|
81
83
|
desc 'version', 'Prints cnvrg current version'
|
84
|
+
|
82
85
|
def version
|
83
86
|
puts Cnvrg::VERSION
|
84
87
|
|
@@ -87,6 +90,7 @@ module Cnvrg
|
|
87
90
|
map %w(-v --version) => :version
|
88
91
|
|
89
92
|
desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
|
93
|
+
|
90
94
|
def set_api_url(url)
|
91
95
|
home_dir = File.expand_path('~')
|
92
96
|
if !url.end_with? "/api"
|
@@ -153,7 +157,7 @@ module Cnvrg
|
|
153
157
|
map %w(-api --api) => :set_api_url
|
154
158
|
|
155
159
|
desc 'set_default_owner', 'set default owner'
|
156
|
-
|
160
|
+
|
157
161
|
def set_default_owner
|
158
162
|
|
159
163
|
path = File.expand_path('~')+"/.cnvrg/config.yml"
|
@@ -177,7 +181,7 @@ module Cnvrg
|
|
177
181
|
owners << choose_owner
|
178
182
|
chosen = false
|
179
183
|
while !chosen
|
180
|
-
owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
|
184
|
+
owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
|
181
185
|
choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
|
182
186
|
|
183
187
|
if choose_owner =~ /[[:digit:]]/
|
@@ -250,7 +254,7 @@ module Cnvrg
|
|
250
254
|
owners << choose_owner
|
251
255
|
chosen = false
|
252
256
|
while !chosen
|
253
|
-
owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
|
257
|
+
owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
|
254
258
|
choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
|
255
259
|
|
256
260
|
if choose_owner =~ /[[:digit:]]/
|
@@ -417,7 +421,7 @@ module Cnvrg
|
|
417
421
|
say "created project successfully", Thor::Shell::Color::GREEN
|
418
422
|
say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
|
419
423
|
rescue => e
|
420
|
-
log_end(-1,e.message)
|
424
|
+
log_end(-1, e.message)
|
421
425
|
say "Error occurred, aborting", Thor::Shell::Color::RED
|
422
426
|
if Dir.exist? working_dir
|
423
427
|
|
@@ -436,6 +440,7 @@ module Cnvrg
|
|
436
440
|
exit(1)
|
437
441
|
end
|
438
442
|
end
|
443
|
+
|
439
444
|
desc 'set_image', 'set_image for a project'
|
440
445
|
|
441
446
|
def set_image(docker_image)
|
@@ -472,6 +477,7 @@ module Cnvrg
|
|
472
477
|
@image.update_image_activity(nil, nil)
|
473
478
|
|
474
479
|
end
|
480
|
+
|
475
481
|
desc 'link', 'Link current directory to a new cnvrg project'
|
476
482
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
477
483
|
method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
|
@@ -498,7 +504,7 @@ module Cnvrg
|
|
498
504
|
end
|
499
505
|
working_dir = Dir.getwd
|
500
506
|
owner = CLI.get_owner
|
501
|
-
if Project.link(owner,project_name, docker)
|
507
|
+
if Project.link(owner, project_name, docker)
|
502
508
|
path = Dir.pwd
|
503
509
|
@project = Project.new(path)
|
504
510
|
@project.generate_idx()
|
@@ -556,10 +562,10 @@ module Cnvrg
|
|
556
562
|
end
|
557
563
|
end
|
558
564
|
|
559
|
-
desc '
|
565
|
+
desc 'data init', 'Init dataset directory'
|
560
566
|
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
561
567
|
|
562
|
-
def init_data
|
568
|
+
def init_data(public)
|
563
569
|
begin
|
564
570
|
verify_logged_in(false)
|
565
571
|
log_start(__method__, args, options)
|
@@ -574,9 +580,10 @@ module Cnvrg
|
|
574
580
|
|
575
581
|
working_dir = Dir.getwd
|
576
582
|
owner = CLI.get_owner
|
577
|
-
if Dataset.init(owner,dataset_name, options["public"])
|
583
|
+
if Dataset.init(owner, dataset_name, options["public"])
|
578
584
|
path = Dir.pwd
|
579
585
|
@dataset = Dataset.new(path)
|
586
|
+
|
580
587
|
@dataset.generate_idx()
|
581
588
|
|
582
589
|
url = @dataset.url
|
@@ -598,7 +605,117 @@ module Cnvrg
|
|
598
605
|
exit(1)
|
599
606
|
end
|
600
607
|
end
|
601
|
-
|
608
|
+
|
609
|
+
desc 'data clone', 'Clone dataset'
|
610
|
+
method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
|
611
|
+
|
612
|
+
def clone_data(dataset_url)
|
613
|
+
begin
|
614
|
+
verify_logged_in(false)
|
615
|
+
log_start(__method__, args, options)
|
616
|
+
url_parts = dataset_url.split("/")
|
617
|
+
project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
|
618
|
+
slug = url_parts[project_index+1]
|
619
|
+
owner = url_parts[project_index-1]
|
620
|
+
response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
|
621
|
+
|
622
|
+
Cnvrg::CLI.is_response_success(response)
|
623
|
+
dataset_name = response["result"]["name"]
|
624
|
+
if (Dir.exists? dataset_name)
|
625
|
+
say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
|
626
|
+
if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
|
627
|
+
say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
|
628
|
+
log_end(1, "conflict with dir #{dataset_name}")
|
629
|
+
|
630
|
+
exit(1)
|
631
|
+
end
|
632
|
+
|
633
|
+
end
|
634
|
+
if Dataset.clone(owner, dataset_name, slug)
|
635
|
+
say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
|
636
|
+
|
637
|
+
commit_to_clone = options["commit"] || nil
|
638
|
+
working_dir = "#{Dir.pwd}/#{dataset_name}"
|
639
|
+
@dataset = Dataset.new(working_dir)
|
640
|
+
@dataset.generate_idx()
|
641
|
+
say "Downloading data", Thor::Shell::Color::BLUE
|
642
|
+
|
643
|
+
|
644
|
+
download_data(false, false, path = working_dir)
|
645
|
+
|
646
|
+
|
647
|
+
check = Helpers.checkmark
|
648
|
+
say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
|
649
|
+
log_end(0)
|
650
|
+
|
651
|
+
else
|
652
|
+
log_end(1, "can't create dataset")
|
653
|
+
@dataset.revert(working_dir) unless @dataset.nil?
|
654
|
+
say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
|
655
|
+
exit(0)
|
656
|
+
end
|
657
|
+
rescue SignalException
|
658
|
+
log_end(-1)
|
659
|
+
|
660
|
+
say "\nAborting"
|
661
|
+
exit(1)
|
662
|
+
end
|
663
|
+
end
|
664
|
+
|
665
|
+
desc 'init_data_container', 'Init dataset directory', :hide => true
|
666
|
+
method_option :container, :type => :string, :aliases => ["--c"], :default => ""
|
667
|
+
method_option :login_content, :type => :string, :aliases => ["--l"], :default => ""
|
668
|
+
method_option :owner, :type => :string, :aliases => ["--o"], :default => ""
|
669
|
+
method_option :dataset_slug, :type => :string, :aliases => ["--ds"], :default => ""
|
670
|
+
method_option :dataset_name, :type => :string, :aliases => ["--dn"], :default => ""
|
671
|
+
method_option :sha1, :type => :string, :aliases => ["--s"], :default => ""
|
672
|
+
method_option :username, :type => :string, :aliases => ["--u"], :default => ""
|
673
|
+
method_option :api_url, :type => :string, :aliases => ["--a"], :default => ""
|
674
|
+
|
675
|
+
def init_data_container()
|
676
|
+
begin
|
677
|
+
container = options["container"]
|
678
|
+
login_content = options["login_content"]
|
679
|
+
dataset_slug = options["dataset_slug"]
|
680
|
+
owner = options["owner"]
|
681
|
+
dataset_name = options["dataset_name"]
|
682
|
+
sha1 = options["sha1"]
|
683
|
+
username = options["username"]
|
684
|
+
api_url = options["api_url"]
|
685
|
+
|
686
|
+
container = Docker::Container.get(container)
|
687
|
+
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
688
|
+
container.exec(command, tty: true)
|
689
|
+
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
|
690
|
+
container.exec(command, tty: true)
|
691
|
+
command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
|
692
|
+
container.exec(command, tty: true)
|
693
|
+
command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
|
694
|
+
container.exec(command, tty: true)
|
695
|
+
command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
|
696
|
+
container.exec(command, tty: true)
|
697
|
+
command = ["/bin/bash", "-lc", "cnvrg set_remote_api_url #{owner} #{username} #{api_url}"]
|
698
|
+
container.exec(command, tty: true)
|
699
|
+
command = ["/bin/bash", "-lc", "mkdir -p /home/ds/notebooks/data/.cnvrg"]
|
700
|
+
container.exec(command, tty: true)
|
701
|
+
command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrg/config.yml"]
|
702
|
+
container.exec(command, tty: true)
|
703
|
+
command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrgignore"]
|
704
|
+
container.exec(command, tty: true)
|
705
|
+
|
706
|
+
command = ["/bin/bash", "-lc", "cnvrg data_init_container #{owner} #{dataset_slug} #{dataset_name}"]
|
707
|
+
container.exec(command, tty: true)
|
708
|
+
|
709
|
+
|
710
|
+
rescue SignalException
|
711
|
+
log_end(-1)
|
712
|
+
|
713
|
+
say "\nAborting"
|
714
|
+
exit(1)
|
715
|
+
end
|
716
|
+
end
|
717
|
+
|
718
|
+
desc 'data_snap', 'Init dataset directory', :hide => true
|
602
719
|
method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
|
603
720
|
|
604
721
|
def snap_data
|
@@ -607,11 +724,10 @@ module Cnvrg
|
|
607
724
|
log_start(__method__, args, options)
|
608
725
|
|
609
726
|
owner = CLI.get_owner
|
610
|
-
|
611
|
-
|
612
|
-
puts @dataset.snapshot
|
727
|
+
path = Dir.pwd
|
728
|
+
@dataset = Dataset.new(path)
|
613
729
|
|
614
|
-
|
730
|
+
log_end(0)
|
615
731
|
|
616
732
|
rescue SignalException
|
617
733
|
log_end(-1)
|
@@ -620,14 +736,116 @@ module Cnvrg
|
|
620
736
|
exit(1)
|
621
737
|
end
|
622
738
|
end
|
739
|
+
desc 'data_snap', 'Init dataset directory', :hide => true
|
740
|
+
|
741
|
+
def data_init_container(owner,dataset_slug,dataset_name)
|
742
|
+
puts owner
|
743
|
+
puts dataset_name
|
744
|
+
puts dataset_slug
|
745
|
+
|
746
|
+
if Dataset.init_container(owner, dataset_slug, dataset_name)
|
747
|
+
|
748
|
+
say "init finished successfully", Thor::Shell::Color::GREEN
|
749
|
+
log_end(0)
|
750
|
+
|
751
|
+
else
|
752
|
+
log_end(1, "can't create dataset")
|
753
|
+
say "error creating dataset, please contact support.", Thor::Shell::Color::RED
|
754
|
+
exit(0)
|
755
|
+
end
|
756
|
+
end
|
757
|
+
|
758
|
+
desc 'data download', 'pull data'
|
759
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
760
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
761
|
+
|
762
|
+
def download_data(verbose, sync, path=Dir.pwd)
|
763
|
+
begin
|
764
|
+
verify_logged_in(true)
|
765
|
+
log_start(__method__, args, options)
|
766
|
+
if path.nil? or path.empty?
|
767
|
+
path = Dir.pwd
|
768
|
+
end
|
769
|
+
dataset_dir = is_cnvrg_dir(path)
|
770
|
+
@dataset = Dataset.new(dataset_dir)
|
771
|
+
|
772
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
773
|
+
new_branch = options["new_branch"] || false
|
774
|
+
|
775
|
+
res = @dataset.compare_idx(new_branch)["result"]
|
776
|
+
|
777
|
+
result = res["tree"]
|
778
|
+
|
779
|
+
commit = res["commit"]
|
780
|
+
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
781
|
+
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
782
|
+
log_end(0)
|
783
|
+
return true
|
784
|
+
end
|
785
|
+
result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
|
786
|
+
|
787
|
+
delete = result["result"]["delete"]
|
788
|
+
commits = result["result"]["commits"]
|
789
|
+
updated_idx = result["result"]["idx"]
|
790
|
+
commits.each do |c|
|
791
|
+
file_name = @files.download_data_file(c, dataset_dir)
|
792
|
+
|
793
|
+
if file_name.eql? false or file_name.nil?
|
794
|
+
##RETRY
|
795
|
+
next
|
796
|
+
end
|
797
|
+
file_path = "#{dataset_dir}/#{file_name}"
|
798
|
+
success = extarct_tar(file_path, dataset_dir)
|
799
|
+
if !success
|
800
|
+
#TODO: ?
|
801
|
+
end
|
802
|
+
|
803
|
+
FileUtils.rm_rf([file_path])
|
804
|
+
|
805
|
+
end
|
806
|
+
to_delete = []
|
807
|
+
delete.each do |d|
|
808
|
+
to_delete << "#{dataset_dir}/#{d}"
|
809
|
+
end
|
810
|
+
FileUtils.rm_rf(to_delete)
|
811
|
+
|
812
|
+
@dataset.update_idx(updated_idx)
|
813
|
+
|
814
|
+
|
815
|
+
# result["conflicts"].each do |f|
|
816
|
+
# relative_path = f.gsub(/^#{@dataset.local_path}/, "")
|
817
|
+
# if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
|
818
|
+
# successful_changes << relative_path
|
819
|
+
# end
|
820
|
+
#
|
821
|
+
# end
|
822
|
+
# update idx with latest commit
|
823
|
+
# @dataset.update_idx_with_commit!(commit)
|
824
|
+
check = Helpers.checkmark()
|
825
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
826
|
+
|
827
|
+
|
828
|
+
log_end(0)
|
829
|
+
end
|
830
|
+
rescue
|
831
|
+
log_end(-1)
|
832
|
+
|
833
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
834
|
+
exit(1)
|
835
|
+
rescue SignalException
|
836
|
+
log_end(-1)
|
837
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
838
|
+
exit(1)
|
839
|
+
end
|
840
|
+
|
623
841
|
|
624
|
-
desc 'upload_data', 'Upload data files', :hide=>true
|
842
|
+
desc 'upload_data', 'Upload data files', :hide => true
|
625
843
|
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
626
844
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
|
627
845
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
628
846
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
629
847
|
|
630
|
-
def upload_data(sync=false,direct=false)
|
848
|
+
def upload_data(sync=false, direct=false)
|
631
849
|
|
632
850
|
begin
|
633
851
|
verify_logged_in(true)
|
@@ -643,12 +861,12 @@ module Cnvrg
|
|
643
861
|
result = @dataset.compare_idx(false)
|
644
862
|
|
645
863
|
commit = result["result"]["commit"]
|
646
|
-
|
647
|
-
|
864
|
+
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
865
|
+
log_end(0)
|
648
866
|
|
649
|
-
|
650
|
-
|
651
|
-
|
867
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
868
|
+
exit(1)
|
869
|
+
end
|
652
870
|
|
653
871
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
654
872
|
result = result["result"]["tree"]
|
@@ -745,7 +963,7 @@ module Cnvrg
|
|
745
963
|
exit(0)
|
746
964
|
end
|
747
965
|
if !result["deleted"].nil? and !result["deleted"].empty?
|
748
|
-
update_count +=
|
966
|
+
update_count += result["deleted"].size
|
749
967
|
end
|
750
968
|
if update_count == update_total
|
751
969
|
res = @files.end_commit(commit_sha1)
|
@@ -798,8 +1016,152 @@ module Cnvrg
|
|
798
1016
|
|
799
1017
|
@files.rollback_commit(commit_sha1)
|
800
1018
|
end
|
801
|
-
rescue =>e
|
1019
|
+
rescue => e
|
1020
|
+
|
1021
|
+
log_end(-1)
|
1022
|
+
|
1023
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
1024
|
+
@files.rollback_commit(commit_sha1)
|
1025
|
+
exit(1)
|
1026
|
+
rescue SignalException
|
1027
|
+
log_end(-1)
|
1028
|
+
|
1029
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
1030
|
+
say "\nRolling back all changes", Thor::Shell::Color::BLUE
|
1031
|
+
@files.rollback_commit(commit_sha1)
|
1032
|
+
exit(1)
|
1033
|
+
end
|
1034
|
+
|
1035
|
+
end
|
1036
|
+
|
1037
|
+
desc 'data upload', 'push data'
|
1038
|
+
method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
|
1039
|
+
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1040
|
+
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1041
|
+
|
1042
|
+
def upload_data_tar(ignore, verbose, sync)
|
1043
|
+
|
1044
|
+
begin
|
1045
|
+
verify_logged_in(true)
|
1046
|
+
log_start(__method__, args, options)
|
1047
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1048
|
+
|
1049
|
+
@dataset = Dataset.new(dataset_dir)
|
1050
|
+
|
1051
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
1052
|
+
if !@dataset.update_ignore_list(ignore)
|
1053
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
1054
|
+
end
|
1055
|
+
result = @dataset.compare_idx(false)
|
1056
|
+
|
1057
|
+
|
1058
|
+
commit = result["result"]["commit"]
|
1059
|
+
if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
1060
|
+
log_end(0)
|
1061
|
+
|
1062
|
+
say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
|
1063
|
+
exit(1)
|
1064
|
+
end
|
1065
|
+
|
1066
|
+
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
|
1067
|
+
result = result["result"]["tree"]
|
1068
|
+
check = Helpers.checkmark()
|
1069
|
+
|
1070
|
+
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
1071
|
+
log_end(0)
|
1072
|
+
say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
|
1073
|
+
return true
|
1074
|
+
end
|
1075
|
+
update_count = 0
|
1076
|
+
update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
|
1077
|
+
successful_updates = []
|
1078
|
+
successful_deletions = []
|
1079
|
+
if verbose
|
1080
|
+
if update_total == 1
|
1081
|
+
say "Updating #{update_total} file", Thor::Shell::Color::BLUE
|
1082
|
+
else
|
1083
|
+
say "Updating #{update_total} files", Thor::Shell::Color::BLUE
|
1084
|
+
end
|
1085
|
+
else
|
1086
|
+
say "Syncing files", Thor::Shell::Color::BLUE unless sync
|
1087
|
+
|
1088
|
+
end
|
1089
|
+
|
1090
|
+
# Start commit
|
1091
|
+
res = @files.start_commit(false)["result"]
|
1092
|
+
commit_sha1 =res["commit_sha1"]
|
1093
|
+
commit_time = res["commit_time"]
|
1094
|
+
# upload / update
|
1095
|
+
begin
|
1096
|
+
(result["added"] + result["updated_on_local"]).each do |f|
|
1097
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1098
|
+
successful_updates<< relative_path
|
1099
|
+
update_count += 1
|
1100
|
+
end
|
1101
|
+
|
1102
|
+
# delete
|
1103
|
+
deleted = update_deleted(result["deleted"])
|
1104
|
+
deleted.each do |f|
|
1105
|
+
relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
|
1106
|
+
successful_updates<< relative_path
|
1107
|
+
end
|
1108
|
+
@dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
|
1109
|
+
|
1110
|
+
@dataset.update_idx_with_commit!(commit_sha1)
|
1111
|
+
|
1112
|
+
home_dir = File.expand_path('~')
|
1113
|
+
tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
|
1114
|
+
tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
|
1115
|
+
tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
|
1116
|
+
File.open(tar_files_path, 'w') {|f| f.write tar_files}
|
1117
|
+
create_tar(dataset_dir, tar_path, tar_files_path)
|
1118
|
+
res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
|
1119
|
+
cur_idx = @dataset.get_idx.to_h
|
1120
|
+
res = @files.end_commit_tar(commit_sha1, cur_idx)
|
1121
|
+
|
1122
|
+
# delete
|
1123
|
+
FileUtils.rm_rf([tar_path,tar_files_path])
|
1124
|
+
|
1125
|
+
rescue SignalException
|
1126
|
+
log_end(-1)
|
1127
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1128
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1129
|
+
|
1130
|
+
|
1131
|
+
@files.rollback_commit(commit_sha1)
|
1132
|
+
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
1133
|
+
exit(0)
|
1134
|
+
rescue => e
|
1135
|
+
FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
|
1136
|
+
FileUtils.rm_rf([tar_path]) if File.exist? tar_path
|
1137
|
+
|
1138
|
+
log_end(-1, e.message)
|
1139
|
+
@files.rollback_commit(commit_sha1)
|
1140
|
+
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
1141
|
+
exit(0)
|
1142
|
+
end
|
1143
|
+
if verbose
|
1144
|
+
say "#{check} Done", Thor::Shell::Color::BLUE
|
1145
|
+
if successful_updates.size >0
|
1146
|
+
say "Updated:", Thor::Shell::Color::GREEN
|
1147
|
+
suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1148
|
+
say suc.join("\n"), Thor::Shell::Color::GREEN
|
1149
|
+
end
|
1150
|
+
if successful_deletions.size >0
|
1151
|
+
say "Deleted:", Thor::Shell::Color::GREEN
|
1152
|
+
del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
|
1153
|
+
say del.join("\n"), Thor::Shell::Color::GREEN
|
1154
|
+
end
|
1155
|
+
|
1156
|
+
|
1157
|
+
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
1158
|
+
else
|
1159
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
1160
|
+
|
1161
|
+
end
|
1162
|
+
|
802
1163
|
|
1164
|
+
rescue => e
|
803
1165
|
log_end(-1)
|
804
1166
|
|
805
1167
|
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
@@ -814,10 +1176,13 @@ module Cnvrg
|
|
814
1176
|
exit(1)
|
815
1177
|
end
|
816
1178
|
|
1179
|
+
log_end(0)
|
1180
|
+
|
817
1181
|
end
|
818
1182
|
|
819
1183
|
|
820
|
-
desc 'unlink','Unlink a project from current directory', :hide=>true
|
1184
|
+
desc 'unlink', 'Unlink a project from current directory', :hide => true
|
1185
|
+
|
821
1186
|
def create_volume
|
822
1187
|
verify_logged_in(false)
|
823
1188
|
log_start(__method__, args, options)
|
@@ -827,8 +1192,9 @@ module Cnvrg
|
|
827
1192
|
|
828
1193
|
end
|
829
1194
|
|
830
|
-
desc 'list data','
|
831
|
-
|
1195
|
+
desc 'list data', 'List all dataset you currently have'
|
1196
|
+
|
1197
|
+
def list_dataset
|
832
1198
|
verify_logged_in(true)
|
833
1199
|
log_start(__method__, args, options)
|
834
1200
|
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
@@ -840,13 +1206,29 @@ module Cnvrg
|
|
840
1206
|
|
841
1207
|
end
|
842
1208
|
|
1209
|
+
desc 'list data commits', 'List all commits for a specific dataset'
|
1210
|
+
|
1211
|
+
def list_dataset_commits()
|
1212
|
+
verify_logged_in(true)
|
1213
|
+
log_start(__method__, args, options)
|
1214
|
+
|
1215
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1216
|
+
@dataset = Dataset.new(dataset_dir)
|
1217
|
+
result = @dataset.list_commits()
|
1218
|
+
list = result["result"]["list"]
|
1219
|
+
|
1220
|
+
print_table(list)
|
1221
|
+
|
1222
|
+
end
|
1223
|
+
|
1224
|
+
|
1225
|
+
desc 'unlink', 'Unlink a project from current directory'
|
843
1226
|
|
844
|
-
desc 'unlink','Unlink a project from current directory'
|
845
1227
|
def unlink
|
846
1228
|
verify_logged_in(false)
|
847
1229
|
log_start(__method__, args, options)
|
848
1230
|
working_dir = is_cnvrg_dir()
|
849
|
-
list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
|
1231
|
+
list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
|
850
1232
|
FileUtils.rm_rf list_to_del
|
851
1233
|
end
|
852
1234
|
|
@@ -1024,7 +1406,9 @@ module Cnvrg
|
|
1024
1406
|
exit(1)
|
1025
1407
|
end
|
1026
1408
|
end
|
1027
|
-
|
1409
|
+
|
1410
|
+
desc '', '', :hide => true
|
1411
|
+
|
1028
1412
|
def revert_exp
|
1029
1413
|
begin
|
1030
1414
|
log_start(__method__, args, options)
|
@@ -1033,7 +1417,7 @@ module Cnvrg
|
|
1033
1417
|
result = @project.compare_idx(false)["result"]
|
1034
1418
|
result = result["tree"]
|
1035
1419
|
if result["added"].size > 0
|
1036
|
-
|
1420
|
+
FileUtils.rm_rf(result["added"])
|
1037
1421
|
end
|
1038
1422
|
|
1039
1423
|
rescue SignalException
|
@@ -1045,12 +1429,12 @@ module Cnvrg
|
|
1045
1429
|
|
1046
1430
|
|
1047
1431
|
desc 'upload', 'Upload updated files'
|
1048
|
-
method_option :ignore, :type => :
|
1432
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
1049
1433
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1050
1434
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1051
1435
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1052
1436
|
|
1053
|
-
def upload(link=false, sync=false,direct=false)
|
1437
|
+
def upload(link=false, sync=false, direct=false, ignore_list="")
|
1054
1438
|
|
1055
1439
|
begin
|
1056
1440
|
verify_logged_in(true)
|
@@ -1059,7 +1443,19 @@ module Cnvrg
|
|
1059
1443
|
@project = Project.new(get_project_home)
|
1060
1444
|
|
1061
1445
|
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
1062
|
-
ignore = options[:ignore] ||
|
1446
|
+
ignore = options[:ignore] || ""
|
1447
|
+
|
1448
|
+
if ignore.nil? or ignore.empty?
|
1449
|
+
ignore = ignore_list
|
1450
|
+
end
|
1451
|
+
data_ignore = data_dir_include()
|
1452
|
+
if !data_ignore.nil?
|
1453
|
+
if ignore.nil? or ignore.empty?
|
1454
|
+
ignore = data_ignore
|
1455
|
+
else
|
1456
|
+
ignore ="#{ignore},#{data_ignore}"
|
1457
|
+
end
|
1458
|
+
end
|
1063
1459
|
if !@project.update_ignore_list(ignore)
|
1064
1460
|
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
1065
1461
|
end
|
@@ -1072,7 +1468,6 @@ module Cnvrg
|
|
1072
1468
|
end
|
1073
1469
|
end
|
1074
1470
|
result = @project.compare_idx(new_branch)
|
1075
|
-
|
1076
1471
|
commit = result["result"]["commit"]
|
1077
1472
|
if !link
|
1078
1473
|
if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
@@ -1177,7 +1572,7 @@ module Cnvrg
|
|
1177
1572
|
exit(0)
|
1178
1573
|
end
|
1179
1574
|
if !result["deleted"].nil? and !result["deleted"].empty?
|
1180
|
-
update_count +=
|
1575
|
+
update_count += result["deleted"].size
|
1181
1576
|
end
|
1182
1577
|
if update_count == update_total
|
1183
1578
|
res = @files.end_commit(commit_sha1)
|
@@ -1235,8 +1630,7 @@ module Cnvrg
|
|
1235
1630
|
|
1236
1631
|
@files.rollback_commit(commit_sha1)
|
1237
1632
|
end
|
1238
|
-
rescue =>e
|
1239
|
-
|
1633
|
+
rescue => e
|
1240
1634
|
log_end(-1)
|
1241
1635
|
|
1242
1636
|
say "Error occurd, \nAborting", Thor::Shell::Color::RED
|
@@ -1257,14 +1651,30 @@ module Cnvrg
|
|
1257
1651
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1258
1652
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1259
1653
|
method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
|
1654
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
1260
1655
|
|
1261
|
-
def download(sync=false)
|
1656
|
+
def download(sync=false, ignore_list="")
|
1262
1657
|
begin
|
1263
1658
|
verify_logged_in(true)
|
1264
1659
|
log_start(__method__, args, options)
|
1265
1660
|
project_home = get_project_home
|
1266
1661
|
@project = Project.new(project_home)
|
1267
1662
|
@files = Cnvrg::Files.new(@project.owner, @project.slug)
|
1663
|
+
ignore = options[:ignore] || ""
|
1664
|
+
if ignore.nil? or ignore.empty?
|
1665
|
+
ignore = ignore_list
|
1666
|
+
end
|
1667
|
+
data_ignore = data_dir_include()
|
1668
|
+
if !data_ignore.nil?
|
1669
|
+
if ignore.nil? or ignore.empty?
|
1670
|
+
ignore = data_ignore
|
1671
|
+
else
|
1672
|
+
ignore ="#{ignore},#{data_ignore}"
|
1673
|
+
end
|
1674
|
+
end
|
1675
|
+
if !@project.update_ignore_list(ignore)
|
1676
|
+
say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
|
1677
|
+
end
|
1268
1678
|
new_branch = options["new_branch"] || false
|
1269
1679
|
|
1270
1680
|
res = @project.compare_idx(new_branch)["result"]
|
@@ -1347,7 +1757,7 @@ module Cnvrg
|
|
1347
1757
|
|
1348
1758
|
log_end(0)
|
1349
1759
|
end
|
1350
|
-
rescue
|
1760
|
+
rescue =>e
|
1351
1761
|
log_end(-1)
|
1352
1762
|
|
1353
1763
|
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
@@ -1453,21 +1863,74 @@ module Cnvrg
|
|
1453
1863
|
end
|
1454
1864
|
|
1455
1865
|
|
1866
|
+
desc 'data_jump', 'jump to specific commit', :hide => true
|
1867
|
+
|
1868
|
+
def data_jump(*commit_sha1)
|
1869
|
+
begin
|
1870
|
+
verify_logged_in()
|
1871
|
+
log_start(__method__, args, options)
|
1872
|
+
dataset_dir = is_cnvrg_dir(Dir.pwd)
|
1873
|
+
@dataset = Dataset.new(dataset_dir)
|
1874
|
+
|
1875
|
+
@files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
|
1876
|
+
if commit_sha1.nil? or commit_sha1.empty?
|
1877
|
+
commit_sha1 = @dataset.last_local_commit
|
1878
|
+
end
|
1879
|
+
response = @dataset.compare_commits(commit_sha1)
|
1880
|
+
successful_changes = []
|
1881
|
+
if !response["result"]["status"].nil?
|
1882
|
+
idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
|
1883
|
+
File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
|
1884
|
+
status = response["result"]["status"]
|
1885
|
+
(status["delete"]).each do |f|
|
1886
|
+
relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
|
1887
|
+
FileUtils.rm_rf(relative_path)
|
1888
|
+
end
|
1889
|
+
# current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
|
1890
|
+
(status["dirs"]).each do |f|
|
1891
|
+
relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
|
1892
|
+
# dir
|
1893
|
+
if @files.download_dir(dataset_dir, relative_path)
|
1894
|
+
# current_tree.delete(relative_path[0, relative_path.size-1])
|
1895
|
+
successful_changes << relative_path
|
1896
|
+
end
|
1897
|
+
end
|
1898
|
+
(status["download"]).each do |f|
|
1899
|
+
relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
|
1900
|
+
# dir
|
1901
|
+
if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
|
1902
|
+
successful_changes << relative_path
|
1903
|
+
end
|
1904
|
+
end
|
1905
|
+
|
1906
|
+
|
1907
|
+
say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
|
1908
|
+
log_end(0)
|
1909
|
+
end
|
1910
|
+
|
1911
|
+
rescue SignalException
|
1912
|
+
log_end(-1)
|
1913
|
+
exi(1)
|
1914
|
+
end
|
1915
|
+
end
|
1916
|
+
|
1456
1917
|
desc 'sync', 'Sync with remote server'
|
1457
1918
|
method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
|
1458
1919
|
method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
|
1920
|
+
method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
|
1459
1921
|
|
1460
1922
|
def sync(direct=true)
|
1461
1923
|
verify_logged_in(true) if direct
|
1462
1924
|
if options["verbose"]
|
1463
|
-
|
1925
|
+
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
1464
1926
|
else
|
1465
1927
|
say 'Syncing project', Thor::Shell::Color::BLUE
|
1466
1928
|
end
|
1467
1929
|
|
1468
1930
|
|
1469
|
-
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
|
1470
|
-
invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"]
|
1931
|
+
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
|
1932
|
+
invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
|
1933
|
+
:ignore => options[:ignore]
|
1471
1934
|
|
1472
1935
|
|
1473
1936
|
end
|
@@ -1476,8 +1939,8 @@ module Cnvrg
|
|
1476
1939
|
desc 'run cmd', 'Runs an experiment'
|
1477
1940
|
method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
|
1478
1941
|
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
1479
|
-
method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
|
1480
|
-
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
1942
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
1943
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
1481
1944
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1482
1945
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
1483
1946
|
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
@@ -1491,6 +1954,9 @@ module Cnvrg
|
|
1491
1954
|
method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
|
1492
1955
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1493
1956
|
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
1957
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
1958
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
1959
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
1494
1960
|
|
1495
1961
|
def run(*cmd)
|
1496
1962
|
sync_before = options["sync_before"]
|
@@ -1504,31 +1970,41 @@ module Cnvrg
|
|
1504
1970
|
schedule = options["schedule"]
|
1505
1971
|
image = options["image"]
|
1506
1972
|
grid = options["grid"]
|
1973
|
+
data = options["data"]
|
1974
|
+
data_commit = options["data_commit"]
|
1975
|
+
ignore = options["ignore"]
|
1976
|
+
|
1507
1977
|
if local
|
1508
1978
|
invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
|
1509
|
-
:log => log, :email_notification => email_notification, :upload_output => upload_output,
|
1979
|
+
:log => log, :email_notification => email_notification, :upload_output => upload_output,
|
1980
|
+
:commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
|
1510
1981
|
return
|
1511
1982
|
else
|
1512
1983
|
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1513
|
-
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
1984
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
1514
1985
|
instance_type = get_instance_type(instances)
|
1515
1986
|
invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
1516
|
-
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
|
1987
|
+
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
|
1988
|
+
:image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
|
1517
1989
|
return
|
1518
1990
|
end
|
1519
1991
|
end
|
1520
1992
|
|
1521
1993
|
|
1522
|
-
desc '', ''
|
1994
|
+
desc '', '', :hide => true
|
1523
1995
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
|
1524
1996
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
|
1525
1997
|
method_option :title, :type => :string, :aliases => ["-t"], :default => ""
|
1526
1998
|
method_option :log, :type => :boolean, :aliases => ["-l", "--log"], :default => false
|
1527
1999
|
method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
|
1528
|
-
method_option :upload_output, :type => :string, :aliases => [
|
2000
|
+
method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
|
1529
2001
|
method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
|
1530
2002
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1531
|
-
method_option :indocker, :type => :boolean,
|
2003
|
+
method_option :indocker, :type => :boolean, :default => false
|
2004
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2005
|
+
method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
|
2006
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
2007
|
+
|
1532
2008
|
|
1533
2009
|
def exec(*cmd)
|
1534
2010
|
log = []
|
@@ -1537,7 +2013,7 @@ module Cnvrg
|
|
1537
2013
|
verify_logged_in(true)
|
1538
2014
|
log_start(__method__, args, options)
|
1539
2015
|
working_dir = is_cnvrg_dir
|
1540
|
-
script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
|
2016
|
+
script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
|
1541
2017
|
|
1542
2018
|
|
1543
2019
|
sync_before = options["sync_before"]
|
@@ -1545,8 +2021,10 @@ module Cnvrg
|
|
1545
2021
|
print_log = options["log"]
|
1546
2022
|
title = options["title"]
|
1547
2023
|
commit = options["commit"] || nil
|
1548
|
-
image = options["image"]
|
1549
|
-
indocker = options["indocker"]
|
2024
|
+
image = options["image"] || nil
|
2025
|
+
indocker = options["indocker"] || false
|
2026
|
+
ignore = options[:ignore] || ""
|
2027
|
+
|
1550
2028
|
|
1551
2029
|
email_notification = options["email_notification"]
|
1552
2030
|
upload_output = options["upload_output"]
|
@@ -1561,7 +2039,8 @@ module Cnvrg
|
|
1561
2039
|
else
|
1562
2040
|
if sync_before
|
1563
2041
|
# Sync before run
|
1564
|
-
|
2042
|
+
|
2043
|
+
invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
|
1565
2044
|
end
|
1566
2045
|
end
|
1567
2046
|
#set image for the project
|
@@ -1569,27 +2048,27 @@ module Cnvrg
|
|
1569
2048
|
invoke :set_image, [image]
|
1570
2049
|
end
|
1571
2050
|
if !indocker
|
1572
|
-
|
2051
|
+
image_proj = is_project_with_docker(working_dir)
|
1573
2052
|
|
1574
2053
|
|
1575
|
-
|
1576
|
-
|
1577
|
-
|
1578
|
-
|
1579
|
-
|
1580
|
-
|
2054
|
+
if image_proj and image_proj.is_docker
|
2055
|
+
container= image_proj.get_container
|
2056
|
+
if !container
|
2057
|
+
say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
|
2058
|
+
exit(1)
|
2059
|
+
end
|
1581
2060
|
|
1582
2061
|
|
1583
|
-
|
1584
|
-
|
1585
|
-
|
1586
|
-
|
1587
|
-
|
1588
|
-
|
1589
|
-
|
1590
|
-
|
1591
|
-
|
1592
|
-
|
2062
|
+
exec_args = args.flatten.join(" ")
|
2063
|
+
options_hash = Hash[options]
|
2064
|
+
options_hash.except!("image", "indocker")
|
2065
|
+
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
2066
|
+
command_to_run = cmd.join(" ")
|
2067
|
+
command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
|
2068
|
+
puts container.exec(command, tty: true)
|
2069
|
+
container.stop()
|
2070
|
+
exit(0)
|
2071
|
+
end
|
1593
2072
|
end
|
1594
2073
|
start_commit = @project.last_local_commit
|
1595
2074
|
cmd = cmd.join("\s")
|
@@ -1602,7 +2081,7 @@ module Cnvrg
|
|
1602
2081
|
machine_name = Socket.gethostname
|
1603
2082
|
begin
|
1604
2083
|
machine_activity = @exp.get_machine_activity(working_dir)
|
1605
|
-
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
|
2084
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
|
1606
2085
|
unless @exp.slug.nil?
|
1607
2086
|
real = Time.now
|
1608
2087
|
exp_success = true
|
@@ -1660,7 +2139,7 @@ module Cnvrg
|
|
1660
2139
|
end
|
1661
2140
|
|
1662
2141
|
rescue Errno::EIO => e
|
1663
|
-
|
2142
|
+
# break
|
1664
2143
|
rescue Errno::ENOENT
|
1665
2144
|
log_end(1, "command #{cmd} isn't valid")
|
1666
2145
|
|
@@ -1712,8 +2191,8 @@ module Cnvrg
|
|
1712
2191
|
# Sync after run
|
1713
2192
|
|
1714
2193
|
|
1715
|
-
download(sync=true)
|
1716
|
-
upload(link=false, sync=true,direct=false)
|
2194
|
+
download(sync=true, ignore_list=ignore)
|
2195
|
+
upload(link=false, sync=true, direct=false, ignore_list=ignore)
|
1717
2196
|
|
1718
2197
|
end
|
1719
2198
|
end_commit = @project.last_local_commit
|
@@ -1790,7 +2269,7 @@ module Cnvrg
|
|
1790
2269
|
machine_name = Socket.gethostname
|
1791
2270
|
begin
|
1792
2271
|
|
1793
|
-
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
|
2272
|
+
@exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
|
1794
2273
|
unless @exp.slug.nil?
|
1795
2274
|
real = Time.now
|
1796
2275
|
exp_success = true
|
@@ -1944,19 +2423,37 @@ module Cnvrg
|
|
1944
2423
|
method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
|
1945
2424
|
method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
|
1946
2425
|
method_option :machine_type, :type => :string, :default => ""
|
1947
|
-
method_option :schedule, :type => :string, :aliases => [
|
1948
|
-
method_option :commit, :type => :string, :aliases => [
|
2426
|
+
method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
|
2427
|
+
method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
|
1949
2428
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1950
2429
|
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
2430
|
+
method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
|
2431
|
+
method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
|
2432
|
+
method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
|
2433
|
+
|
1951
2434
|
|
1952
2435
|
def exec_remote(*cmd)
|
1953
2436
|
verify_logged_in(true)
|
1954
2437
|
log_start(__method__, args, options)
|
1955
2438
|
working_dir = is_cnvrg_dir
|
1956
|
-
path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
|
2439
|
+
path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
|
1957
2440
|
|
1958
2441
|
begin
|
1959
2442
|
grid = options["grid"] || nil
|
2443
|
+
data = options["data"] || nil
|
2444
|
+
data_commit = options["data_commit"] || nil
|
2445
|
+
ignore = options[:ignore] || ""
|
2446
|
+
|
2447
|
+
if !data.nil? and !data.empty?
|
2448
|
+
if ignore.nil? or ignore.empty?
|
2449
|
+
ignore = "data"
|
2450
|
+
else
|
2451
|
+
ignore +=",data"
|
2452
|
+
end
|
2453
|
+
end
|
2454
|
+
if ignore.nil? or ignore.empty?
|
2455
|
+
ignore= ""
|
2456
|
+
end
|
1960
2457
|
|
1961
2458
|
instance_type = options["machine_type"] || nil
|
1962
2459
|
schedule = options["schedule"] || ""
|
@@ -1986,9 +2483,9 @@ module Cnvrg
|
|
1986
2483
|
upload_output_option = "--upload_output=#{upload_output}"
|
1987
2484
|
end
|
1988
2485
|
options_hash = Hash[options]
|
1989
|
-
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid")
|
2486
|
+
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "ignore", "data", "data_commit")
|
1990
2487
|
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
1991
|
-
command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
2488
|
+
command = "#{exec_options} --ignore=#{ignore} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
1992
2489
|
commit_to_run = options["commit"] || nil
|
1993
2490
|
if !schedule.nil? and !schedule.empty?
|
1994
2491
|
|
@@ -1999,7 +2496,7 @@ module Cnvrg
|
|
1999
2496
|
choose_image = options["image"]
|
2000
2497
|
|
2001
2498
|
if !choose_image.nil? and !choose_image.empty?
|
2002
|
-
invoke :set_image,[choose_image]
|
2499
|
+
invoke :set_image, [choose_image]
|
2003
2500
|
end
|
2004
2501
|
image = is_project_with_docker(working_dir)
|
2005
2502
|
if !image or !image.is_docker
|
@@ -2021,15 +2518,13 @@ module Cnvrg
|
|
2021
2518
|
end
|
2022
2519
|
|
2023
2520
|
|
2024
|
-
|
2025
2521
|
invoke :sync, [false], []
|
2026
2522
|
|
2027
2523
|
say "Running remote experiment", Thor::Shell::Color::BLUE
|
2028
2524
|
exp = Experiment.new(project.owner, project.slug)
|
2029
2525
|
|
2030
|
-
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
|
2526
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
|
2031
2527
|
if Cnvrg::CLI.is_response_success(res)
|
2032
|
-
|
2033
2528
|
# if res["result"]["machine"] == -1
|
2034
2529
|
# say "There are no available machines", Thor::Shell::Color::BLUE
|
2035
2530
|
# create = yes? "create new machine?", Thor::Shell::Color::YELLOW
|
@@ -2061,7 +2556,7 @@ module Cnvrg
|
|
2061
2556
|
# end
|
2062
2557
|
# else
|
2063
2558
|
check = Helpers.checkmark()
|
2064
|
-
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
2559
|
+
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
2065
2560
|
|
2066
2561
|
if res["result"]["grid"]
|
2067
2562
|
str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
|
@@ -2101,7 +2596,6 @@ module Cnvrg
|
|
2101
2596
|
schedule = options["schedule"] || ""
|
2102
2597
|
|
2103
2598
|
|
2104
|
-
|
2105
2599
|
if !schedule.nil? and !schedule.empty?
|
2106
2600
|
|
2107
2601
|
local_timestamp = get_schedule_date
|
@@ -2126,7 +2620,6 @@ module Cnvrg
|
|
2126
2620
|
end
|
2127
2621
|
|
2128
2622
|
|
2129
|
-
|
2130
2623
|
invoke :sync, [false], []
|
2131
2624
|
|
2132
2625
|
res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
@@ -2186,15 +2679,16 @@ module Cnvrg
|
|
2186
2679
|
method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
|
2187
2680
|
method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
2188
2681
|
method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
|
2189
|
-
method_option :small, :type => :boolean, :aliases => ["-sm","--small"], :default => false
|
2190
|
-
method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
|
2191
|
-
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
2682
|
+
method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
|
2683
|
+
method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
|
2684
|
+
method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
|
2192
2685
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
2193
2686
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
2194
2687
|
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
2195
2688
|
method_option :image, :type => :string, :aliases => ["--i"], :default => ""
|
2196
2689
|
|
2197
2690
|
desc 'notebook', 'starts a notebook session remotely or locally'
|
2691
|
+
|
2198
2692
|
def notebook
|
2199
2693
|
local = options["local"]
|
2200
2694
|
notebook_dir = options["notebook_dir"]
|
@@ -2202,14 +2696,14 @@ module Cnvrg
|
|
2202
2696
|
image = options["image"]
|
2203
2697
|
|
2204
2698
|
if local
|
2205
|
-
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel
|
2699
|
+
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
|
2206
2700
|
return
|
2207
2701
|
else
|
2208
2702
|
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
2209
|
-
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
2703
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
|
2210
2704
|
instance_type = get_instance_type(instances)
|
2211
2705
|
|
2212
|
-
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image =>image
|
2706
|
+
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image
|
2213
2707
|
return
|
2214
2708
|
|
2215
2709
|
end
|
@@ -2219,7 +2713,7 @@ module Cnvrg
|
|
2219
2713
|
|
2220
2714
|
desc 'remote_notebook', 'run notebook server on remote server'
|
2221
2715
|
method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
2222
|
-
method_option :machine_type, :type => :string,
|
2716
|
+
method_option :machine_type, :type => :string, :default => ""
|
2223
2717
|
method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
|
2224
2718
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
2225
2719
|
|
@@ -2239,7 +2733,7 @@ module Cnvrg
|
|
2239
2733
|
choose_image = options["image"]
|
2240
2734
|
|
2241
2735
|
if !choose_image.nil? and !choose_image.empty?
|
2242
|
-
invoke :set_image,[choose_image]
|
2736
|
+
invoke :set_image, [choose_image]
|
2243
2737
|
end
|
2244
2738
|
@image = is_project_with_docker(working_dir)
|
2245
2739
|
if !@image or !@image.is_docker
|
@@ -2250,7 +2744,7 @@ module Cnvrg
|
|
2250
2744
|
end
|
2251
2745
|
# default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
|
2252
2746
|
# if default
|
2253
|
-
|
2747
|
+
@image = Images.new(working_dir, default_image_name)
|
2254
2748
|
# else
|
2255
2749
|
# exit(0)
|
2256
2750
|
# end
|
@@ -2259,7 +2753,6 @@ module Cnvrg
|
|
2259
2753
|
invoke :sync, [false], []
|
2260
2754
|
|
2261
2755
|
|
2262
|
-
|
2263
2756
|
res = @image.remote_notebook(notebook_dir, instance_type, kernel)
|
2264
2757
|
if Cnvrg::CLI.is_response_success(res)
|
2265
2758
|
if res["result"]["machine"] == -1
|
@@ -2368,7 +2861,7 @@ module Cnvrg
|
|
2368
2861
|
end
|
2369
2862
|
rescue => e
|
2370
2863
|
log_end(-1, e.message)
|
2371
|
-
say "Error occurred, aborting",Thor::Shell::Color::RED
|
2864
|
+
say "Error occurred, aborting", Thor::Shell::Color::RED
|
2372
2865
|
if container
|
2373
2866
|
container.stop()
|
2374
2867
|
end
|
@@ -2473,7 +2966,7 @@ module Cnvrg
|
|
2473
2966
|
choose_image = options["image"]
|
2474
2967
|
|
2475
2968
|
if !choose_image.nil? and !choose_image.empty?
|
2476
|
-
invoke :set_image,[choose_image]
|
2969
|
+
invoke :set_image, [choose_image]
|
2477
2970
|
end
|
2478
2971
|
image = is_project_with_docker(Dir.pwd)
|
2479
2972
|
if image and image.is_docker and !remote
|
@@ -2510,7 +3003,7 @@ module Cnvrg
|
|
2510
3003
|
base_url = get_base_url()
|
2511
3004
|
|
2512
3005
|
local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
|
2513
|
-
command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
3006
|
+
command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
|
2514
3007
|
container.exec(command, tty: true)
|
2515
3008
|
container.stop()
|
2516
3009
|
container.start()
|
@@ -2526,7 +3019,7 @@ module Cnvrg
|
|
2526
3019
|
exit(1)
|
2527
3020
|
end
|
2528
3021
|
|
2529
|
-
url =
|
3022
|
+
url = list[list.size-1].split(" ::")
|
2530
3023
|
token = url[0][(url[0].index("=")+1)..-1]
|
2531
3024
|
|
2532
3025
|
# machine_activity = @note.get_machine_activity(project_dir)
|
@@ -2562,7 +3055,7 @@ module Cnvrg
|
|
2562
3055
|
# exit(1)
|
2563
3056
|
# end
|
2564
3057
|
rescue => e
|
2565
|
-
log_end(-1,e.message)
|
3058
|
+
log_end(-1, e.message)
|
2566
3059
|
say "Error occurd, aborting", Thor::Shell::Color::RED
|
2567
3060
|
if container
|
2568
3061
|
container.stop()
|
@@ -2583,6 +3076,7 @@ module Cnvrg
|
|
2583
3076
|
method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
|
2584
3077
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
|
2585
3078
|
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
3079
|
+
|
2586
3080
|
def notebook_stop
|
2587
3081
|
begin
|
2588
3082
|
verify_logged_in(true)
|
@@ -2638,7 +3132,7 @@ module Cnvrg
|
|
2638
3132
|
exit(0)
|
2639
3133
|
end
|
2640
3134
|
rescue => e
|
2641
|
-
log_end(-1,e.message)
|
3135
|
+
log_end(-1, e.message)
|
2642
3136
|
say "Error occurd, aborting"
|
2643
3137
|
if container
|
2644
3138
|
container.stop()
|
@@ -2688,7 +3182,7 @@ module Cnvrg
|
|
2688
3182
|
|
2689
3183
|
log_end(0)
|
2690
3184
|
rescue => e
|
2691
|
-
log_end(-1,e.message)
|
3185
|
+
log_end(-1, e.message)
|
2692
3186
|
say "Error occurd, aborting"
|
2693
3187
|
if container
|
2694
3188
|
container.stop()
|
@@ -2751,7 +3245,7 @@ module Cnvrg
|
|
2751
3245
|
|
2752
3246
|
log_end(0)
|
2753
3247
|
rescue => e
|
2754
|
-
log_end(-1,e.message)
|
3248
|
+
log_end(-1, e.message)
|
2755
3249
|
say "Error occurd, aborting"
|
2756
3250
|
if container
|
2757
3251
|
container.stop()
|
@@ -2766,6 +3260,7 @@ module Cnvrg
|
|
2766
3260
|
end
|
2767
3261
|
|
2768
3262
|
end
|
3263
|
+
|
2769
3264
|
desc 'build', 'run commands inside containers'
|
2770
3265
|
method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
|
2771
3266
|
|
@@ -2777,10 +3272,10 @@ module Cnvrg
|
|
2777
3272
|
working_dir = is_cnvrg_dir
|
2778
3273
|
install_file = options["install"] || nil
|
2779
3274
|
if !install_file.nil?
|
2780
|
-
commands = File.open(install_file).read.chop.gsub!("\n",",").split(",")
|
3275
|
+
commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
|
2781
3276
|
|
2782
3277
|
else
|
2783
|
-
commands
|
3278
|
+
commands = [cmd.join(" ")]
|
2784
3279
|
end
|
2785
3280
|
|
2786
3281
|
|
@@ -2798,25 +3293,25 @@ module Cnvrg
|
|
2798
3293
|
|
2799
3294
|
end
|
2800
3295
|
commands.each do |c|
|
2801
|
-
|
2802
|
-
|
2803
|
-
|
2804
|
-
|
2805
|
-
|
2806
|
-
|
3296
|
+
if c.include? "pip"
|
3297
|
+
c.sub("pip", "/opt/ds/bin/pip")
|
3298
|
+
end
|
3299
|
+
if c.include? "pip3"
|
3300
|
+
c.sub("pip3", "/opt/ds3/bin/pip3")
|
3301
|
+
end
|
2807
3302
|
|
2808
|
-
|
2809
|
-
|
2810
|
-
|
2811
|
-
|
2812
|
-
|
2813
|
-
|
2814
|
-
|
3303
|
+
say "Running #{c}", Thor::Shell::Color::BLUE
|
3304
|
+
command = ["/bin/bash", "-lc", "#{c}"]
|
3305
|
+
res = container.exec(command, tty: false)
|
3306
|
+
if res[2] != 0
|
3307
|
+
say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
|
3308
|
+
container.stop()
|
3309
|
+
log_end(0)
|
2815
3310
|
|
2816
|
-
|
2817
|
-
|
2818
|
-
|
2819
|
-
|
3311
|
+
exit(1)
|
3312
|
+
end
|
3313
|
+
say res[0].join("\n")
|
3314
|
+
image.store_image_build_commands(working_dir, c)
|
2820
3315
|
end
|
2821
3316
|
|
2822
3317
|
checks = Helpers.checkmark()
|
@@ -2827,7 +3322,7 @@ module Cnvrg
|
|
2827
3322
|
|
2828
3323
|
log_end(0)
|
2829
3324
|
rescue => e
|
2830
|
-
log_end(-1,e.message)
|
3325
|
+
log_end(-1, e.message)
|
2831
3326
|
say "Error occurd, aborting", Thor::Shell::Color::RED
|
2832
3327
|
if container
|
2833
3328
|
container.stop()
|
@@ -2876,7 +3371,7 @@ module Cnvrg
|
|
2876
3371
|
log_end(0)
|
2877
3372
|
return new_image.id
|
2878
3373
|
rescue => e
|
2879
|
-
log_end(-1,e.message)
|
3374
|
+
log_end(-1, e.message)
|
2880
3375
|
say "\nError occurred,aborting"
|
2881
3376
|
exit(1)
|
2882
3377
|
rescue SignalException
|
@@ -2939,22 +3434,22 @@ module Cnvrg
|
|
2939
3434
|
end
|
2940
3435
|
if !name.nil? and !name.empty?
|
2941
3436
|
if name.include? " "
|
2942
|
-
name.gsub!(" ","_")
|
3437
|
+
name.gsub!(" ", "_")
|
2943
3438
|
end
|
2944
3439
|
end
|
2945
|
-
stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n",",")
|
3440
|
+
stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
|
2946
3441
|
if stored_commands.nil? or stored_commands.empty?
|
2947
3442
|
say "Nothing to push", Thor::Shell::Color::BLUE
|
2948
3443
|
exit(0)
|
2949
3444
|
end
|
2950
3445
|
|
2951
3446
|
say "Pushing new image", Thor::Shell::Color::BLUE
|
2952
|
-
if image.create_custom_image(name,working_dir,stored_commands)
|
3447
|
+
if image.create_custom_image(name, working_dir, stored_commands)
|
2953
3448
|
|
2954
3449
|
say "Image was updated successfully", Thor::Shell::Color::GREEN
|
2955
3450
|
end
|
2956
3451
|
rescue => e
|
2957
|
-
log_end(-1,e.message)
|
3452
|
+
log_end(-1, e.message)
|
2958
3453
|
say "error occurred, aborting"
|
2959
3454
|
|
2960
3455
|
end
|
@@ -3018,7 +3513,7 @@ module Cnvrg
|
|
3018
3513
|
exit(1)
|
3019
3514
|
end
|
3020
3515
|
rescue => e
|
3021
|
-
log_end(-1,e.message)
|
3516
|
+
log_end(-1, e.message)
|
3022
3517
|
say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
|
3023
3518
|
rescue SignalException
|
3024
3519
|
log_end(-1)
|
@@ -3044,9 +3539,8 @@ module Cnvrg
|
|
3044
3539
|
container.start()
|
3045
3540
|
cnvrg_command = cmd.join(" ")
|
3046
3541
|
command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
|
3047
|
-
res = container.exec(command, tty: true,wait:5400)[0]
|
3542
|
+
res = container.exec(command, tty: true, wait: 5400)[0]
|
3048
3543
|
say res
|
3049
|
-
|
3050
3544
|
end
|
3051
3545
|
|
3052
3546
|
desc '', '', :hide => true
|
@@ -3054,8 +3548,13 @@ module Cnvrg
|
|
3054
3548
|
def port_container(container_id)
|
3055
3549
|
container = Docker::Container.get(container_id)
|
3056
3550
|
say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
|
3551
|
+
end
|
3057
3552
|
|
3553
|
+
desc '', '', :hide => true
|
3058
3554
|
|
3555
|
+
def tensor_port_container(container_id)
|
3556
|
+
container = Docker::Container.get(container_id)
|
3557
|
+
say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
|
3059
3558
|
end
|
3060
3559
|
|
3061
3560
|
desc '', '', :hide => true
|
@@ -3071,10 +3570,9 @@ module Cnvrg
|
|
3071
3570
|
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
3072
3571
|
method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
|
3073
3572
|
method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
3074
|
-
method_option :volume, :type => :string, :aliases => ["-v", "--v"], :default => "/home/ds/notebooks/data"
|
3075
3573
|
|
3076
3574
|
|
3077
|
-
def config_remote(image_name, port=7654,tensport=6006)
|
3575
|
+
def config_remote(image_name, port=7654, tensport=6006)
|
3078
3576
|
local_images = Docker::Image.all
|
3079
3577
|
|
3080
3578
|
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
|
@@ -3097,7 +3595,6 @@ module Cnvrg
|
|
3097
3595
|
'ExposedPorts' => {
|
3098
3596
|
'8888/tcp' => {},
|
3099
3597
|
},
|
3100
|
-
'VolumesFrom' =>volume_from,
|
3101
3598
|
'HostConfig' => {
|
3102
3599
|
'PortBindings' => {
|
3103
3600
|
'8888/tcp' => [
|
@@ -3124,7 +3621,7 @@ module Cnvrg
|
|
3124
3621
|
say "#{container.id}:#{port}##{tensport}"
|
3125
3622
|
rescue => e
|
3126
3623
|
if e.message.include? "is not running"
|
3127
|
-
return config_remote(image_name, port-1,tensport-1)
|
3624
|
+
return config_remote(image_name, port-1, tensport-1)
|
3128
3625
|
end
|
3129
3626
|
|
3130
3627
|
if container
|
@@ -3133,13 +3630,14 @@ module Cnvrg
|
|
3133
3630
|
return false
|
3134
3631
|
end
|
3135
3632
|
end
|
3633
|
+
|
3136
3634
|
desc '', '', :hide => true
|
3137
3635
|
method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
|
3138
3636
|
method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
|
3139
3637
|
method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
|
3140
3638
|
|
3141
3639
|
|
3142
|
-
def config_remote_gpu(image_name, port=7654,tensport=6006)
|
3640
|
+
def config_remote_gpu(image_name, port=7654, tensport=6006)
|
3143
3641
|
local_images = Docker::Image.all
|
3144
3642
|
|
3145
3643
|
docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
|
@@ -3174,7 +3672,7 @@ module Cnvrg
|
|
3174
3672
|
# }
|
3175
3673
|
|
3176
3674
|
container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} #{image_name}:latest #{cmd} `
|
3177
|
-
container_id = container_id.gsub("\n","")
|
3675
|
+
container_id = container_id.gsub("\n", "")
|
3178
3676
|
container = Docker::Container.get(container_id)
|
3179
3677
|
# container.start()
|
3180
3678
|
command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
|
@@ -3191,7 +3689,7 @@ module Cnvrg
|
|
3191
3689
|
rescue => e
|
3192
3690
|
if e.message.include? "is not running"
|
3193
3691
|
puts "running asgain with: #{port-1} #{tensport-1}"
|
3194
|
-
return config_remote_gpu(image_name, port-1,tensport-1)
|
3692
|
+
return config_remote_gpu(image_name, port-1, tensport-1)
|
3195
3693
|
end
|
3196
3694
|
|
3197
3695
|
if container
|
@@ -3344,7 +3842,6 @@ module Cnvrg
|
|
3344
3842
|
end
|
3345
3843
|
|
3346
3844
|
|
3347
|
-
|
3348
3845
|
return dir_path
|
3349
3846
|
|
3350
3847
|
else
|
@@ -3378,13 +3875,13 @@ module Cnvrg
|
|
3378
3875
|
owner = Cnvrg::CLI.get_owner()
|
3379
3876
|
res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
|
3380
3877
|
if Cnvrg::CLI.is_response_success(res)
|
3381
|
-
printf "%-20s %-20s %-30s %-20s %-20s\n", "name", "project",
|
3878
|
+
printf "%-20s %-20s %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
|
3382
3879
|
res["result"]["images"].each do |u|
|
3383
3880
|
time = Time.parse(u["created_at"])
|
3384
3881
|
update_at = get_local_time(time)
|
3385
3882
|
created_by = u["created_by"]
|
3386
3883
|
|
3387
|
-
printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"],
|
3884
|
+
printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
|
3388
3885
|
end
|
3389
3886
|
end
|
3390
3887
|
log_end(0)
|
@@ -3728,12 +4225,13 @@ module Cnvrg
|
|
3728
4225
|
return owner
|
3729
4226
|
end
|
3730
4227
|
end
|
4228
|
+
|
3731
4229
|
def get_base_url
|
3732
4230
|
home_dir = File.expand_path('~')
|
3733
4231
|
|
3734
4232
|
config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
|
3735
4233
|
api = config.to_h[:api]
|
3736
|
-
return api.gsub!("/api","")
|
4234
|
+
return api.gsub!("/api", "")
|
3737
4235
|
end
|
3738
4236
|
|
3739
4237
|
def get_project_home
|
@@ -3857,6 +4355,18 @@ module Cnvrg
|
|
3857
4355
|
return false
|
3858
4356
|
end
|
3859
4357
|
end
|
4358
|
+
def data_dir_include()
|
4359
|
+
all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
|
4360
|
+
all_dirs.flatten!
|
4361
|
+
all_dirs.each do |a|
|
4362
|
+
if a.include? "/.cnvrg"
|
4363
|
+
ignore = File.dirname(a)
|
4364
|
+
return ignore
|
4365
|
+
end
|
4366
|
+
end
|
4367
|
+
return nil
|
4368
|
+
|
4369
|
+
end
|
3860
4370
|
|
3861
4371
|
def verify_software_installed(software)
|
3862
4372
|
begin
|
@@ -4032,20 +4542,21 @@ module Cnvrg
|
|
4032
4542
|
return new_time
|
4033
4543
|
|
4034
4544
|
end
|
4545
|
+
|
4035
4546
|
def update_deleted(deleted)
|
4036
4547
|
final = []
|
4037
4548
|
deleted.each do |d|
|
4038
|
-
|
4039
|
-
|
4040
|
-
|
4041
|
-
|
4042
|
-
|
4043
|
-
|
4044
|
-
|
4045
|
-
end
|
4046
|
-
value = value+"/"+all_subs[i+1] if i <all_subs.size-1
|
4549
|
+
all_subs = d.split("/")
|
4550
|
+
to_add = true
|
4551
|
+
value = all_subs[0]
|
4552
|
+
all_subs.each_with_index do |a, i|
|
4553
|
+
if final.include? value+"/"
|
4554
|
+
to_add = false
|
4555
|
+
break
|
4047
4556
|
end
|
4048
|
-
|
4557
|
+
value = value+"/"+all_subs[i+1] if i <all_subs.size-1
|
4558
|
+
end
|
4559
|
+
final << d if to_add
|
4049
4560
|
|
4050
4561
|
|
4051
4562
|
end
|
@@ -4053,7 +4564,8 @@ module Cnvrg
|
|
4053
4564
|
|
4054
4565
|
return final
|
4055
4566
|
end
|
4056
|
-
|
4567
|
+
|
4568
|
+
def get_cmd_path_in_dir(main_dir, sub_dir)
|
4057
4569
|
first = Pathname.new main_dir
|
4058
4570
|
second = Pathname.new sub_dir
|
4059
4571
|
relative = second.relative_path_from first
|
@@ -4064,10 +4576,18 @@ module Cnvrg
|
|
4064
4576
|
end
|
4065
4577
|
end
|
4066
4578
|
|
4579
|
+
def create_tar(path_in, path_out, tar_files)
|
4580
|
+
`tar -czf #{path_out} -T #{tar_files} > /dev/null 2>&1`
|
4581
|
+
return $?.success?
|
4582
|
+
end
|
4067
4583
|
|
4068
|
-
|
4584
|
+
def extarct_tar(file_path, dir_path)
|
4585
|
+
`tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
|
4586
|
+
return $?.success?
|
4587
|
+
end
|
4069
4588
|
end
|
4070
4589
|
end
|
4071
4590
|
end
|
4072
4591
|
|
4073
4592
|
|
4593
|
+
|