cnvrg 0.0.148 → 0.0.149

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9889a94a7cbc7cf8e729daf363e61bd38072960a
4
- data.tar.gz: e46f0fed0b9b6a3007daa399dc4e75b81d402aa6
3
+ metadata.gz: 89c404e15ae2c15390c06632e16fdb46725a7dab
4
+ data.tar.gz: 28435401db7d5d2a78ba88de8946513ffc86dca5
5
5
  SHA512:
6
- metadata.gz: 8bc2c1dca91ad6f2ff9e4248b95b34231bef97be4eb9d7ba52b5cd73c9def6ee36ad672beaf45c8581e868d0b89fc7d8cf6e6f6babfd35ed6f5feb5bf610c222
7
- data.tar.gz: 02d1c07bec47cada6285f3df813b6fc67b0c3f2700df70bc89be691e72f20593da9672ce2fa03430dc2d548e5ba9ac930a7f3d2711b2a36e6fe32f022367e589
6
+ metadata.gz: 5442ff1a286b258d5624ba1bc00d7944423a99b0b74128cd1b90dffd24511e52e9609a7119ba2d24c4bec348927f7a637fb669b7f961d780f6070103e8cb8e20
7
+ data.tar.gz: 678610c32a2104cab60cec8bcab410cc5d76372079b99044776e444cc7713f206cb10b87521425459e3a8c53937032665a795fd36adfa48a58c9227a1963b998
@@ -33,11 +33,13 @@ require 'active_support/all'
33
33
  require 'thor'
34
34
  require 'pathname'
35
35
  require 'cnvrg/data'
36
-
37
36
  # DEV VERSION
38
37
 
39
- #
38
+ # include Cnvrg::Tar
39
+
40
+ # include Tar
40
41
  module Cnvrg
42
+
41
43
  class CLI < Thor
42
44
 
43
45
  INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
@@ -50,10 +52,11 @@ module Cnvrg
50
52
  super
51
53
  end
52
54
  end
53
- desc "data", "upload and manage datasets", :hide =>true
55
+ desc "data", "upload and manage datasets", :hide => true
54
56
  subcommand "data", Data
55
57
 
56
- desc "", "" , :hide => true
58
+ desc "", "", :hide => true
59
+
57
60
  def test
58
61
  # image_settings = {
59
62
  # 'Image' => "cnvrg:latest",
@@ -74,11 +77,11 @@ module Cnvrg
74
77
  # s = "/leah/1/2/3/4/5"
75
78
  # command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{s}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
76
79
  # puts container.exec(command, tty: true)
77
- puts File.read("s4cmd.py")
78
80
  end
79
81
 
80
82
 
81
83
  desc 'version', 'Prints cnvrg current version'
84
+
82
85
  def version
83
86
  puts Cnvrg::VERSION
84
87
 
@@ -87,6 +90,7 @@ module Cnvrg
87
90
  map %w(-v --version) => :version
88
91
 
89
92
  desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
93
+
90
94
  def set_api_url(url)
91
95
  home_dir = File.expand_path('~')
92
96
  if !url.end_with? "/api"
@@ -153,7 +157,7 @@ module Cnvrg
153
157
  map %w(-api --api) => :set_api_url
154
158
 
155
159
  desc 'set_default_owner', 'set default owner'
156
-
160
+
157
161
  def set_default_owner
158
162
 
159
163
  path = File.expand_path('~')+"/.cnvrg/config.yml"
@@ -177,7 +181,7 @@ module Cnvrg
177
181
  owners << choose_owner
178
182
  chosen = false
179
183
  while !chosen
180
- owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
184
+ owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
181
185
  choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
182
186
 
183
187
  if choose_owner =~ /[[:digit:]]/
@@ -250,7 +254,7 @@ module Cnvrg
250
254
  owners << choose_owner
251
255
  chosen = false
252
256
  while !chosen
253
- owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
257
+ owners_id = owners.each_with_index.map { |x, i| "#{i+1}. #{x}" }
254
258
  choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
255
259
 
256
260
  if choose_owner =~ /[[:digit:]]/
@@ -417,7 +421,7 @@ module Cnvrg
417
421
  say "created project successfully", Thor::Shell::Color::GREEN
418
422
  say "Linked directory to\t#{@project.url}", Thor::Shell::Color::GREEN
419
423
  rescue => e
420
- log_end(-1,e.message)
424
+ log_end(-1, e.message)
421
425
  say "Error occurred, aborting", Thor::Shell::Color::RED
422
426
  if Dir.exist? working_dir
423
427
 
@@ -436,6 +440,7 @@ module Cnvrg
436
440
  exit(1)
437
441
  end
438
442
  end
443
+
439
444
  desc 'set_image', 'set_image for a project'
440
445
 
441
446
  def set_image(docker_image)
@@ -472,6 +477,7 @@ module Cnvrg
472
477
  @image.update_image_activity(nil, nil)
473
478
 
474
479
  end
480
+
475
481
  desc 'link', 'Link current directory to a new cnvrg project'
476
482
  method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
477
483
  method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
@@ -498,7 +504,7 @@ module Cnvrg
498
504
  end
499
505
  working_dir = Dir.getwd
500
506
  owner = CLI.get_owner
501
- if Project.link(owner,project_name, docker)
507
+ if Project.link(owner, project_name, docker)
502
508
  path = Dir.pwd
503
509
  @project = Project.new(path)
504
510
  @project.generate_idx()
@@ -556,10 +562,10 @@ module Cnvrg
556
562
  end
557
563
  end
558
564
 
559
- desc 'init_data', 'Init dataset directory', :hide =>true
565
+ desc 'data init', 'Init dataset directory'
560
566
  method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
561
567
 
562
- def init_data
568
+ def init_data(public)
563
569
  begin
564
570
  verify_logged_in(false)
565
571
  log_start(__method__, args, options)
@@ -574,9 +580,10 @@ module Cnvrg
574
580
 
575
581
  working_dir = Dir.getwd
576
582
  owner = CLI.get_owner
577
- if Dataset.init(owner,dataset_name, options["public"])
583
+ if Dataset.init(owner, dataset_name, options["public"])
578
584
  path = Dir.pwd
579
585
  @dataset = Dataset.new(path)
586
+
580
587
  @dataset.generate_idx()
581
588
 
582
589
  url = @dataset.url
@@ -598,7 +605,117 @@ module Cnvrg
598
605
  exit(1)
599
606
  end
600
607
  end
601
- desc 'data_snap', 'Init dataset directory', :hide =>true
608
+
609
+ desc 'data clone', 'Clone dataset'
610
+ method_option :commit, :type => :string, :aliases => ["-c", "--commit"], :default => ""
611
+
612
+ def clone_data(dataset_url)
613
+ begin
614
+ verify_logged_in(false)
615
+ log_start(__method__, args, options)
616
+ url_parts = dataset_url.split("/")
617
+ project_index = Cnvrg::Helpers.look_for_in_path(dataset_url, "datasets")
618
+ slug = url_parts[project_index+1]
619
+ owner = url_parts[project_index-1]
620
+ response = Cnvrg::API.request("users/#{owner}/datasets/#{slug}/clone", 'GET')
621
+
622
+ Cnvrg::CLI.is_response_success(response)
623
+ dataset_name = response["result"]["name"]
624
+ if (Dir.exists? dataset_name)
625
+ say "Error: Conflict with dir #{dataset_name}", Thor::Shell::Color::RED
626
+ if no? "Sync to repository anyway? (current data might lost)", Thor::Shell::Color::YELLOW
627
+ say "Remove dir in order to clone #{dataset_name}", Thor::Shell::Color::RED
628
+ log_end(1, "conflict with dir #{dataset_name}")
629
+
630
+ exit(1)
631
+ end
632
+
633
+ end
634
+ if Dataset.clone(owner, dataset_name, slug)
635
+ say "Cloning #{dataset_name}", Thor::Shell::Color::BLUE
636
+
637
+ commit_to_clone = options["commit"] || nil
638
+ working_dir = "#{Dir.pwd}/#{dataset_name}"
639
+ @dataset = Dataset.new(working_dir)
640
+ @dataset.generate_idx()
641
+ say "Downloading data", Thor::Shell::Color::BLUE
642
+
643
+
644
+ download_data(false, false, path = working_dir)
645
+
646
+
647
+ check = Helpers.checkmark
648
+ say "#{check} Clone finished successfully", Thor::Shell::Color::GREEN
649
+ log_end(0)
650
+
651
+ else
652
+ log_end(1, "can't create dataset")
653
+ @dataset.revert(working_dir) unless @dataset.nil?
654
+ say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
655
+ exit(0)
656
+ end
657
+ rescue SignalException
658
+ log_end(-1)
659
+
660
+ say "\nAborting"
661
+ exit(1)
662
+ end
663
+ end
664
+
665
+ desc 'init_data_container', 'Init dataset directory', :hide => true
666
+ method_option :container, :type => :string, :aliases => ["--c"], :default => ""
667
+ method_option :login_content, :type => :string, :aliases => ["--l"], :default => ""
668
+ method_option :owner, :type => :string, :aliases => ["--o"], :default => ""
669
+ method_option :dataset_slug, :type => :string, :aliases => ["--ds"], :default => ""
670
+ method_option :dataset_name, :type => :string, :aliases => ["--dn"], :default => ""
671
+ method_option :sha1, :type => :string, :aliases => ["--s"], :default => ""
672
+ method_option :username, :type => :string, :aliases => ["--u"], :default => ""
673
+ method_option :api_url, :type => :string, :aliases => ["--a"], :default => ""
674
+
675
+ def init_data_container()
676
+ begin
677
+ container = options["container"]
678
+ login_content = options["login_content"]
679
+ dataset_slug = options["dataset_slug"]
680
+ owner = options["owner"]
681
+ dataset_name = options["dataset_name"]
682
+ sha1 = options["sha1"]
683
+ username = options["username"]
684
+ api_url = options["api_url"]
685
+
686
+ container = Docker::Container.get(container)
687
+ command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
688
+ container.exec(command, tty: true)
689
+ command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg"]
690
+ container.exec(command, tty: true)
691
+ command = ["/bin/bash", "-lc", "mkdir /home/ds/.cnvrg/tmp"]
692
+ container.exec(command, tty: true)
693
+ command = ["/bin/bash", "-lc", "sudo chown -R ds /home/ds/.cnvrg /home/ds/.netrc"]
694
+ container.exec(command, tty: true)
695
+ command = ["/bin/bash", "-lc", "sudo chmod 0600 /home/ds/.netrc"]
696
+ container.exec(command, tty: true)
697
+ command = ["/bin/bash", "-lc", "cnvrg set_remote_api_url #{owner} #{username} #{api_url}"]
698
+ container.exec(command, tty: true)
699
+ command = ["/bin/bash", "-lc", "mkdir -p /home/ds/notebooks/data/.cnvrg"]
700
+ container.exec(command, tty: true)
701
+ command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrg/config.yml"]
702
+ container.exec(command, tty: true)
703
+ command = ["/bin/bash", "-lc", "touch /home/ds/notebooks/data/.cnvrgignore"]
704
+ container.exec(command, tty: true)
705
+
706
+ command = ["/bin/bash", "-lc", "cnvrg data_init_container #{owner} #{dataset_slug} #{dataset_name}"]
707
+ container.exec(command, tty: true)
708
+
709
+
710
+ rescue SignalException
711
+ log_end(-1)
712
+
713
+ say "\nAborting"
714
+ exit(1)
715
+ end
716
+ end
717
+
718
+ desc 'data_snap', 'Init dataset directory', :hide => true
602
719
  method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
603
720
 
604
721
  def snap_data
@@ -607,11 +724,10 @@ module Cnvrg
607
724
  log_start(__method__, args, options)
608
725
 
609
726
  owner = CLI.get_owner
610
- path = Dir.pwd
611
- @dataset = Dataset.new(path)
612
- puts @dataset.snapshot
727
+ path = Dir.pwd
728
+ @dataset = Dataset.new(path)
613
729
 
614
- log_end(0)
730
+ log_end(0)
615
731
 
616
732
  rescue SignalException
617
733
  log_end(-1)
@@ -620,14 +736,116 @@ module Cnvrg
620
736
  exit(1)
621
737
  end
622
738
  end
739
+ desc 'data_snap', 'Init dataset directory', :hide => true
740
+
741
+ def data_init_container(owner,dataset_slug,dataset_name)
742
+ puts owner
743
+ puts dataset_name
744
+ puts dataset_slug
745
+
746
+ if Dataset.init_container(owner, dataset_slug, dataset_name)
747
+
748
+ say "init finished successfully", Thor::Shell::Color::GREEN
749
+ log_end(0)
750
+
751
+ else
752
+ log_end(1, "can't create dataset")
753
+ say "error creating dataset, please contact support.", Thor::Shell::Color::RED
754
+ exit(0)
755
+ end
756
+ end
757
+
758
+ desc 'data download', 'pull data'
759
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
760
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
761
+
762
+ def download_data(verbose, sync, path=Dir.pwd)
763
+ begin
764
+ verify_logged_in(true)
765
+ log_start(__method__, args, options)
766
+ if path.nil? or path.empty?
767
+ path = Dir.pwd
768
+ end
769
+ dataset_dir = is_cnvrg_dir(path)
770
+ @dataset = Dataset.new(dataset_dir)
771
+
772
+ @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
773
+ new_branch = options["new_branch"] || false
774
+
775
+ res = @dataset.compare_idx(new_branch)["result"]
776
+
777
+ result = res["tree"]
778
+
779
+ commit = res["commit"]
780
+ if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
781
+ say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
782
+ log_end(0)
783
+ return true
784
+ end
785
+ result = @dataset.downlowd_updated_data(@dataset.last_local_commit)
786
+
787
+ delete = result["result"]["delete"]
788
+ commits = result["result"]["commits"]
789
+ updated_idx = result["result"]["idx"]
790
+ commits.each do |c|
791
+ file_name = @files.download_data_file(c, dataset_dir)
792
+
793
+ if file_name.eql? false or file_name.nil?
794
+ ##RETRY
795
+ next
796
+ end
797
+ file_path = "#{dataset_dir}/#{file_name}"
798
+ success = extarct_tar(file_path, dataset_dir)
799
+ if !success
800
+ #TODO: ?
801
+ end
802
+
803
+ FileUtils.rm_rf([file_path])
804
+
805
+ end
806
+ to_delete = []
807
+ delete.each do |d|
808
+ to_delete << "#{dataset_dir}/#{d}"
809
+ end
810
+ FileUtils.rm_rf(to_delete)
811
+
812
+ @dataset.update_idx(updated_idx)
813
+
814
+
815
+ # result["conflicts"].each do |f|
816
+ # relative_path = f.gsub(/^#{@dataset.local_path}/, "")
817
+ # if @files.download_file_s3(f, relative_path, dataset_dir, conflict=true)
818
+ # successful_changes << relative_path
819
+ # end
820
+ #
821
+ # end
822
+ # update idx with latest commit
823
+ # @dataset.update_idx_with_commit!(commit)
824
+ check = Helpers.checkmark()
825
+ say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
826
+
827
+
828
+ log_end(0)
829
+ end
830
+ rescue
831
+ log_end(-1)
832
+
833
+ say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
834
+ exit(1)
835
+ rescue SignalException
836
+ log_end(-1)
837
+ say "\nAborting", Thor::Shell::Color::BLUE
838
+ exit(1)
839
+ end
840
+
623
841
 
624
- desc 'upload_data', 'Upload data files', :hide=>true
842
+ desc 'upload_data', 'Upload data files', :hide => true
625
843
  method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
626
844
  method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
627
845
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
628
846
  method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
629
847
 
630
- def upload_data(sync=false,direct=false)
848
+ def upload_data(sync=false, direct=false)
631
849
 
632
850
  begin
633
851
  verify_logged_in(true)
@@ -643,12 +861,12 @@ module Cnvrg
643
861
  result = @dataset.compare_idx(false)
644
862
 
645
863
  commit = result["result"]["commit"]
646
- if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
647
- log_end(0)
864
+ if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
865
+ log_end(0)
648
866
 
649
- say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
650
- exit(1)
651
- end
867
+ say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
868
+ exit(1)
869
+ end
652
870
 
653
871
  say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
654
872
  result = result["result"]["tree"]
@@ -745,7 +963,7 @@ module Cnvrg
745
963
  exit(0)
746
964
  end
747
965
  if !result["deleted"].nil? and !result["deleted"].empty?
748
- update_count += result["deleted"].size
966
+ update_count += result["deleted"].size
749
967
  end
750
968
  if update_count == update_total
751
969
  res = @files.end_commit(commit_sha1)
@@ -798,8 +1016,152 @@ module Cnvrg
798
1016
 
799
1017
  @files.rollback_commit(commit_sha1)
800
1018
  end
801
- rescue =>e
1019
+ rescue => e
1020
+
1021
+ log_end(-1)
1022
+
1023
+ say "Error occurd, \nAborting", Thor::Shell::Color::RED
1024
+ @files.rollback_commit(commit_sha1)
1025
+ exit(1)
1026
+ rescue SignalException
1027
+ log_end(-1)
1028
+
1029
+ say "\nAborting", Thor::Shell::Color::BLUE
1030
+ say "\nRolling back all changes", Thor::Shell::Color::BLUE
1031
+ @files.rollback_commit(commit_sha1)
1032
+ exit(1)
1033
+ end
1034
+
1035
+ end
1036
+
1037
+ desc 'data upload', 'push data'
1038
+ method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
1039
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1040
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
1041
+
1042
+ def upload_data_tar(ignore, verbose, sync)
1043
+
1044
+ begin
1045
+ verify_logged_in(true)
1046
+ log_start(__method__, args, options)
1047
+ dataset_dir = is_cnvrg_dir(Dir.pwd)
1048
+
1049
+ @dataset = Dataset.new(dataset_dir)
1050
+
1051
+ @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
1052
+ if !@dataset.update_ignore_list(ignore)
1053
+ say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
1054
+ end
1055
+ result = @dataset.compare_idx(false)
1056
+
1057
+
1058
+ commit = result["result"]["commit"]
1059
+ if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
1060
+ log_end(0)
1061
+
1062
+ say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
1063
+ exit(1)
1064
+ end
1065
+
1066
+ say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if verbose
1067
+ result = result["result"]["tree"]
1068
+ check = Helpers.checkmark()
1069
+
1070
+ if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
1071
+ log_end(0)
1072
+ say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless (sync)
1073
+ return true
1074
+ end
1075
+ update_count = 0
1076
+ update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
1077
+ successful_updates = []
1078
+ successful_deletions = []
1079
+ if verbose
1080
+ if update_total == 1
1081
+ say "Updating #{update_total} file", Thor::Shell::Color::BLUE
1082
+ else
1083
+ say "Updating #{update_total} files", Thor::Shell::Color::BLUE
1084
+ end
1085
+ else
1086
+ say "Syncing files", Thor::Shell::Color::BLUE unless sync
1087
+
1088
+ end
1089
+
1090
+ # Start commit
1091
+ res = @files.start_commit(false)["result"]
1092
+ commit_sha1 =res["commit_sha1"]
1093
+ commit_time = res["commit_time"]
1094
+ # upload / update
1095
+ begin
1096
+ (result["added"] + result["updated_on_local"]).each do |f|
1097
+ relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
1098
+ successful_updates<< relative_path
1099
+ update_count += 1
1100
+ end
1101
+
1102
+ # delete
1103
+ deleted = update_deleted(result["deleted"])
1104
+ deleted.each do |f|
1105
+ relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
1106
+ successful_updates<< relative_path
1107
+ end
1108
+ @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), commit_time)
1109
+
1110
+ @dataset.update_idx_with_commit!(commit_sha1)
1111
+
1112
+ home_dir = File.expand_path('~')
1113
+ tar_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.tar.gz"
1114
+ tar_files_path = "#{home_dir}/.cnvrg/tmp/#{@dataset.slug}_#{commit_sha1}.txt"
1115
+ tar_files = (result["added"] + result["updated_on_local"] +["\n"]).join("\n")
1116
+ File.open(tar_files_path, 'w') {|f| f.write tar_files}
1117
+ create_tar(dataset_dir, tar_path, tar_files_path)
1118
+ res = @files.upload_tar_file(tar_path, tar_path, commit_sha1)
1119
+ cur_idx = @dataset.get_idx.to_h
1120
+ res = @files.end_commit_tar(commit_sha1, cur_idx)
1121
+
1122
+ # delete
1123
+ FileUtils.rm_rf([tar_path,tar_files_path])
1124
+
1125
+ rescue SignalException
1126
+ log_end(-1)
1127
+ FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
1128
+ FileUtils.rm_rf([tar_path]) if File.exist? tar_path
1129
+
1130
+
1131
+ @files.rollback_commit(commit_sha1)
1132
+ say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
1133
+ exit(0)
1134
+ rescue => e
1135
+ FileUtils.rm_rf([tar_files_path]) if File.exist? tar_files_path
1136
+ FileUtils.rm_rf([tar_path]) if File.exist? tar_path
1137
+
1138
+ log_end(-1, e.message)
1139
+ @files.rollback_commit(commit_sha1)
1140
+ say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
1141
+ exit(0)
1142
+ end
1143
+ if verbose
1144
+ say "#{check} Done", Thor::Shell::Color::BLUE
1145
+ if successful_updates.size >0
1146
+ say "Updated:", Thor::Shell::Color::GREEN
1147
+ suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
1148
+ say suc.join("\n"), Thor::Shell::Color::GREEN
1149
+ end
1150
+ if successful_deletions.size >0
1151
+ say "Deleted:", Thor::Shell::Color::GREEN
1152
+ del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
1153
+ say del.join("\n"), Thor::Shell::Color::GREEN
1154
+ end
1155
+
1156
+
1157
+ say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
1158
+ else
1159
+ say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
1160
+
1161
+ end
1162
+
802
1163
 
1164
+ rescue => e
803
1165
  log_end(-1)
804
1166
 
805
1167
  say "Error occurd, \nAborting", Thor::Shell::Color::RED
@@ -814,10 +1176,13 @@ module Cnvrg
814
1176
  exit(1)
815
1177
  end
816
1178
 
1179
+ log_end(0)
1180
+
817
1181
  end
818
1182
 
819
1183
 
820
- desc 'unlink','Unlink a project from current directory', :hide=>true
1184
+ desc 'unlink', 'Unlink a project from current directory', :hide => true
1185
+
821
1186
  def create_volume
822
1187
  verify_logged_in(false)
823
1188
  log_start(__method__, args, options)
@@ -827,8 +1192,9 @@ module Cnvrg
827
1192
 
828
1193
  end
829
1194
 
830
- desc 'list data','Unlink a project from current directory', :hide=>true
831
- def list_data
1195
+ desc 'list data', 'List all dataset you currently have'
1196
+
1197
+ def list_dataset
832
1198
  verify_logged_in(true)
833
1199
  log_start(__method__, args, options)
834
1200
  dataset_dir = is_cnvrg_dir(Dir.pwd)
@@ -840,13 +1206,29 @@ module Cnvrg
840
1206
 
841
1207
  end
842
1208
 
1209
+ desc 'list data commits', 'List all commits for a specific dataset'
1210
+
1211
+ def list_dataset_commits()
1212
+ verify_logged_in(true)
1213
+ log_start(__method__, args, options)
1214
+
1215
+ dataset_dir = is_cnvrg_dir(Dir.pwd)
1216
+ @dataset = Dataset.new(dataset_dir)
1217
+ result = @dataset.list_commits()
1218
+ list = result["result"]["list"]
1219
+
1220
+ print_table(list)
1221
+
1222
+ end
1223
+
1224
+
1225
+ desc 'unlink', 'Unlink a project from current directory'
843
1226
 
844
- desc 'unlink','Unlink a project from current directory'
845
1227
  def unlink
846
1228
  verify_logged_in(false)
847
1229
  log_start(__method__, args, options)
848
1230
  working_dir = is_cnvrg_dir()
849
- list_to_del = [working_dir+"/.cnvrgignore",working_dir+"/.cnvrg"]
1231
+ list_to_del = [working_dir+"/.cnvrgignore", working_dir+"/.cnvrg"]
850
1232
  FileUtils.rm_rf list_to_del
851
1233
  end
852
1234
 
@@ -1024,7 +1406,9 @@ module Cnvrg
1024
1406
  exit(1)
1025
1407
  end
1026
1408
  end
1027
- desc '', '' , :hide => true
1409
+
1410
+ desc '', '', :hide => true
1411
+
1028
1412
  def revert_exp
1029
1413
  begin
1030
1414
  log_start(__method__, args, options)
@@ -1033,7 +1417,7 @@ module Cnvrg
1033
1417
  result = @project.compare_idx(false)["result"]
1034
1418
  result = result["tree"]
1035
1419
  if result["added"].size > 0
1036
- FileUtils.rm_rf(result["added"])
1420
+ FileUtils.rm_rf(result["added"])
1037
1421
  end
1038
1422
 
1039
1423
  rescue SignalException
@@ -1045,12 +1429,12 @@ module Cnvrg
1045
1429
 
1046
1430
 
1047
1431
  desc 'upload', 'Upload updated files'
1048
- method_option :ignore, :type => :array, :aliases => ["-i"], :desc => "ignore following files"
1432
+ method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
1049
1433
  method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1050
1434
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1051
1435
  method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
1052
1436
 
1053
- def upload(link=false, sync=false,direct=false)
1437
+ def upload(link=false, sync=false, direct=false, ignore_list="")
1054
1438
 
1055
1439
  begin
1056
1440
  verify_logged_in(true)
@@ -1059,7 +1443,19 @@ module Cnvrg
1059
1443
  @project = Project.new(get_project_home)
1060
1444
 
1061
1445
  @files = Cnvrg::Files.new(@project.owner, @project.slug)
1062
- ignore = options[:ignore] || []
1446
+ ignore = options[:ignore] || ""
1447
+
1448
+ if ignore.nil? or ignore.empty?
1449
+ ignore = ignore_list
1450
+ end
1451
+ data_ignore = data_dir_include()
1452
+ if !data_ignore.nil?
1453
+ if ignore.nil? or ignore.empty?
1454
+ ignore = data_ignore
1455
+ else
1456
+ ignore ="#{ignore},#{data_ignore}"
1457
+ end
1458
+ end
1063
1459
  if !@project.update_ignore_list(ignore)
1064
1460
  say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
1065
1461
  end
@@ -1072,7 +1468,6 @@ module Cnvrg
1072
1468
  end
1073
1469
  end
1074
1470
  result = @project.compare_idx(new_branch)
1075
-
1076
1471
  commit = result["result"]["commit"]
1077
1472
  if !link
1078
1473
  if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
@@ -1177,7 +1572,7 @@ module Cnvrg
1177
1572
  exit(0)
1178
1573
  end
1179
1574
  if !result["deleted"].nil? and !result["deleted"].empty?
1180
- update_count += result["deleted"].size
1575
+ update_count += result["deleted"].size
1181
1576
  end
1182
1577
  if update_count == update_total
1183
1578
  res = @files.end_commit(commit_sha1)
@@ -1235,8 +1630,7 @@ module Cnvrg
1235
1630
 
1236
1631
  @files.rollback_commit(commit_sha1)
1237
1632
  end
1238
- rescue =>e
1239
-
1633
+ rescue => e
1240
1634
  log_end(-1)
1241
1635
 
1242
1636
  say "Error occurd, \nAborting", Thor::Shell::Color::RED
@@ -1257,14 +1651,30 @@ module Cnvrg
1257
1651
  method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1258
1652
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1259
1653
  method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
1654
+ method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
1260
1655
 
1261
- def download(sync=false)
1656
+ def download(sync=false, ignore_list="")
1262
1657
  begin
1263
1658
  verify_logged_in(true)
1264
1659
  log_start(__method__, args, options)
1265
1660
  project_home = get_project_home
1266
1661
  @project = Project.new(project_home)
1267
1662
  @files = Cnvrg::Files.new(@project.owner, @project.slug)
1663
+ ignore = options[:ignore] || ""
1664
+ if ignore.nil? or ignore.empty?
1665
+ ignore = ignore_list
1666
+ end
1667
+ data_ignore = data_dir_include()
1668
+ if !data_ignore.nil?
1669
+ if ignore.nil? or ignore.empty?
1670
+ ignore = data_ignore
1671
+ else
1672
+ ignore ="#{ignore},#{data_ignore}"
1673
+ end
1674
+ end
1675
+ if !@project.update_ignore_list(ignore)
1676
+ say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
1677
+ end
1268
1678
  new_branch = options["new_branch"] || false
1269
1679
 
1270
1680
  res = @project.compare_idx(new_branch)["result"]
@@ -1347,7 +1757,7 @@ module Cnvrg
1347
1757
 
1348
1758
  log_end(0)
1349
1759
  end
1350
- rescue
1760
+ rescue =>e
1351
1761
  log_end(-1)
1352
1762
 
1353
1763
  say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
@@ -1453,21 +1863,74 @@ module Cnvrg
1453
1863
  end
1454
1864
 
1455
1865
 
1866
+ desc 'data_jump', 'jump to specific commit', :hide => true
1867
+
1868
+ def data_jump(*commit_sha1)
1869
+ begin
1870
+ verify_logged_in()
1871
+ log_start(__method__, args, options)
1872
+ dataset_dir = is_cnvrg_dir(Dir.pwd)
1873
+ @dataset = Dataset.new(dataset_dir)
1874
+
1875
+ @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
1876
+ if commit_sha1.nil? or commit_sha1.empty?
1877
+ commit_sha1 = @dataset.last_local_commit
1878
+ end
1879
+ response = @dataset.compare_commits(commit_sha1)
1880
+ successful_changes = []
1881
+ if !response["result"]["status"].nil?
1882
+ idx = {commit: response["result"]["commit"], tree: response["result"]["tree"]}
1883
+ File.open(dataset_dir + "/.cnvrg/idx.yml", "w+") { |f| f.write idx.to_yaml }
1884
+ status = response["result"]["status"]
1885
+ (status["delete"]).each do |f|
1886
+ relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
1887
+ FileUtils.rm_rf(relative_path)
1888
+ end
1889
+ # current_tree = Dir.entries(".").reject { |file| file.start_with? '.' }
1890
+ (status["dirs"]).each do |f|
1891
+ relative_path = f[0].gsub(/^#{@dataset.local_path}/, "")
1892
+ # dir
1893
+ if @files.download_dir(dataset_dir, relative_path)
1894
+ # current_tree.delete(relative_path[0, relative_path.size-1])
1895
+ successful_changes << relative_path
1896
+ end
1897
+ end
1898
+ (status["download"]).each do |f|
1899
+ relative_path = f["name"].gsub(/^#{@dataset.local_path}/, "")
1900
+ # dir
1901
+ if @files.download_file_s3(f["name"], relative_path, dataset_dir, f["sha1"])
1902
+ successful_changes << relative_path
1903
+ end
1904
+ end
1905
+
1906
+
1907
+ say "Done. Jumped completed successfully", Thor::Shell::Color::GREEN
1908
+ log_end(0)
1909
+ end
1910
+
1911
+ rescue SignalException
1912
+ log_end(-1)
1913
+ exi(1)
1914
+ end
1915
+ end
1916
+
1456
1917
  desc 'sync', 'Sync with remote server'
1457
1918
  method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1458
1919
  method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1920
+ method_option :ignore, :type => :string, :aliases => ["-i", "--ignore"], :default => ""
1459
1921
 
1460
1922
  def sync(direct=true)
1461
1923
  verify_logged_in(true) if direct
1462
1924
  if options["verbose"]
1463
- say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
1925
+ say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
1464
1926
  else
1465
1927
  say 'Syncing project', Thor::Shell::Color::BLUE
1466
1928
  end
1467
1929
 
1468
1930
 
1469
- invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
1470
- invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
1931
+ invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true
1932
+ invoke :upload, [link=false, sync=true, direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync => true,
1933
+ :ignore => options[:ignore]
1471
1934
 
1472
1935
 
1473
1936
  end
@@ -1476,8 +1939,8 @@ module Cnvrg
1476
1939
  desc 'run cmd', 'Runs an experiment'
1477
1940
  method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
1478
1941
  method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
1479
- method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
1480
- method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
1942
+ method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
1943
+ method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
1481
1944
  method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
1482
1945
  method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
1483
1946
  method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
@@ -1491,6 +1954,9 @@ module Cnvrg
1491
1954
  method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
1492
1955
  method_option :image, :type => :string, :aliases => ["-i"], :default => ""
1493
1956
  method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
1957
+ method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
1958
+ method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
1959
+ method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
1494
1960
 
1495
1961
  def run(*cmd)
1496
1962
  sync_before = options["sync_before"]
@@ -1504,31 +1970,41 @@ module Cnvrg
1504
1970
  schedule = options["schedule"]
1505
1971
  image = options["image"]
1506
1972
  grid = options["grid"]
1973
+ data = options["data"]
1974
+ data_commit = options["data_commit"]
1975
+ ignore = options["ignore"]
1976
+
1507
1977
  if local
1508
1978
  invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
1509
- :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
1979
+ :log => log, :email_notification => email_notification, :upload_output => upload_output,
1980
+ :commit => commit, :image => image, :data => data, :data_commit => data_commit, :ignore => ignore
1510
1981
  return
1511
1982
  else
1512
1983
  instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
1513
- "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
1984
+ "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
1514
1985
  instance_type = get_instance_type(instances)
1515
1986
  invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
1516
- :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image, :grid => grid
1987
+ :schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit,
1988
+ :image => image, :grid => grid, :data => data, :data_commit => data_commit, :ignore => ignore
1517
1989
  return
1518
1990
  end
1519
1991
  end
1520
1992
 
1521
1993
 
1522
- desc '', '' , :hide => true
1994
+ desc '', '', :hide => true
1523
1995
  method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
1524
1996
  method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
1525
1997
  method_option :title, :type => :string, :aliases => ["-t"], :default => ""
1526
1998
  method_option :log, :type => :boolean, :aliases => ["-l", "--log"], :default => false
1527
1999
  method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
1528
- method_option :upload_output, :type => :string, :aliases => [ "-uo"], :default => ""
2000
+ method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
1529
2001
  method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
1530
2002
  method_option :image, :type => :string, :aliases => ["-i"], :default => ""
1531
- method_option :indocker, :type => :boolean, :default => false
2003
+ method_option :indocker, :type => :boolean, :default => false
2004
+ method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
2005
+ method_option :data_commit, :type => :string, :aliases => ["-dc", "--data_commit"], :default => ""
2006
+ method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
2007
+
1532
2008
 
1533
2009
  def exec(*cmd)
1534
2010
  log = []
@@ -1537,7 +2013,7 @@ module Cnvrg
1537
2013
  verify_logged_in(true)
1538
2014
  log_start(__method__, args, options)
1539
2015
  working_dir = is_cnvrg_dir
1540
- script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
2016
+ script_path = get_cmd_path_in_dir(working_dir, Dir.pwd)
1541
2017
 
1542
2018
 
1543
2019
  sync_before = options["sync_before"]
@@ -1545,8 +2021,10 @@ module Cnvrg
1545
2021
  print_log = options["log"]
1546
2022
  title = options["title"]
1547
2023
  commit = options["commit"] || nil
1548
- image = options["image"] || nil
1549
- indocker = options["indocker"] || false
2024
+ image = options["image"] || nil
2025
+ indocker = options["indocker"] || false
2026
+ ignore = options[:ignore] || ""
2027
+
1550
2028
 
1551
2029
  email_notification = options["email_notification"]
1552
2030
  upload_output = options["upload_output"]
@@ -1561,7 +2039,8 @@ module Cnvrg
1561
2039
  else
1562
2040
  if sync_before
1563
2041
  # Sync before run
1564
- invoke :sync, [false], :new_branch => is_new_branch
2042
+
2043
+ invoke :sync, [false], :new_branch => is_new_branch, :ignore => ignore
1565
2044
  end
1566
2045
  end
1567
2046
  #set image for the project
@@ -1569,27 +2048,27 @@ module Cnvrg
1569
2048
  invoke :set_image, [image]
1570
2049
  end
1571
2050
  if !indocker
1572
- image_proj = is_project_with_docker(working_dir)
2051
+ image_proj = is_project_with_docker(working_dir)
1573
2052
 
1574
2053
 
1575
- if image_proj and image_proj.is_docker
1576
- container= image_proj.get_container
1577
- if !container
1578
- say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
1579
- exit(1)
1580
- end
2054
+ if image_proj and image_proj.is_docker
2055
+ container= image_proj.get_container
2056
+ if !container
2057
+ say "Couldn't create container with image #{image_proj.image_name}:#{image_proj.image_tag}", Thor::Shell::Color::RED
2058
+ exit(1)
2059
+ end
1581
2060
 
1582
2061
 
1583
- exec_args = args.flatten.join(" ")
1584
- options_hash = Hash[options]
1585
- options_hash.except!("image", "indocker")
1586
- exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
1587
- command_to_run = cmd.join(" ")
1588
- command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
1589
- puts container.exec(command, tty: true)
1590
- container.stop()
1591
- exit(0)
1592
- end
2062
+ exec_args = args.flatten.join(" ")
2063
+ options_hash = Hash[options]
2064
+ options_hash.except!("image", "indocker")
2065
+ exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
2066
+ command_to_run = cmd.join(" ")
2067
+ command = ["/bin/bash", "-lc", "cnvrg exec --indocker #{exec_options} #{command_to_run} #{exec_args}"]
2068
+ puts container.exec(command, tty: true)
2069
+ container.stop()
2070
+ exit(0)
2071
+ end
1593
2072
  end
1594
2073
  start_commit = @project.last_local_commit
1595
2074
  cmd = cmd.join("\s")
@@ -1602,7 +2081,7 @@ module Cnvrg
1602
2081
  machine_name = Socket.gethostname
1603
2082
  begin
1604
2083
  machine_activity = @exp.get_machine_activity(working_dir)
1605
- @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
2084
+ @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, script_path)
1606
2085
  unless @exp.slug.nil?
1607
2086
  real = Time.now
1608
2087
  exp_success = true
@@ -1660,7 +2139,7 @@ module Cnvrg
1660
2139
  end
1661
2140
 
1662
2141
  rescue Errno::EIO => e
1663
- # break
2142
+ # break
1664
2143
  rescue Errno::ENOENT
1665
2144
  log_end(1, "command #{cmd} isn't valid")
1666
2145
 
@@ -1712,8 +2191,8 @@ module Cnvrg
1712
2191
  # Sync after run
1713
2192
 
1714
2193
 
1715
- download(sync=true)
1716
- upload(link=false, sync=true,direct=false)
2194
+ download(sync=true, ignore_list=ignore)
2195
+ upload(link=false, sync=true, direct=false, ignore_list=ignore)
1717
2196
 
1718
2197
  end
1719
2198
  end_commit = @project.last_local_commit
@@ -1790,7 +2269,7 @@ module Cnvrg
1790
2269
  machine_name = Socket.gethostname
1791
2270
  begin
1792
2271
 
1793
- @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
2272
+ @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity, Dir.pwd)
1794
2273
  unless @exp.slug.nil?
1795
2274
  real = Time.now
1796
2275
  exp_success = true
@@ -1944,19 +2423,37 @@ module Cnvrg
1944
2423
  method_option :email_notification, :type => :boolean, :aliases => ["-en"], :default => false
1945
2424
  method_option :upload_output, :type => :string, :aliases => ["-uo"], :default => ""
1946
2425
  method_option :machine_type, :type => :string, :default => ""
1947
- method_option :schedule, :type => :string, :aliases => [ "-s"], :default => ""
1948
- method_option :commit, :type => :string, :aliases => [ "-c"], :default => nil
2426
+ method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
2427
+ method_option :commit, :type => :string, :aliases => ["-c, --commit"], :default => nil
1949
2428
  method_option :image, :type => :string, :aliases => ["-i"], :default => ""
1950
2429
  method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
2430
+ method_option :data, :type => :string, :aliases => ["-d", "--data"], :default => ""
2431
+ method_option :data_commit, :type => :string, :aliases => ["--data_commit"], :default => ""
2432
+ method_option :ignore, :type => :string, :aliases => ["-i"], :desc => "ignore following files", :default => ""
2433
+
1951
2434
 
1952
2435
  def exec_remote(*cmd)
1953
2436
  verify_logged_in(true)
1954
2437
  log_start(__method__, args, options)
1955
2438
  working_dir = is_cnvrg_dir
1956
- path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
2439
+ path_to_cmd = get_cmd_path_in_dir(working_dir, Dir.pwd)
1957
2440
 
1958
2441
  begin
1959
2442
  grid = options["grid"] || nil
2443
+ data = options["data"] || nil
2444
+ data_commit = options["data_commit"] || nil
2445
+ ignore = options[:ignore] || ""
2446
+
2447
+ if !data.nil? and !data.empty?
2448
+ if ignore.nil? or ignore.empty?
2449
+ ignore = "data"
2450
+ else
2451
+ ignore +=",data"
2452
+ end
2453
+ end
2454
+ if ignore.nil? or ignore.empty?
2455
+ ignore= ""
2456
+ end
1960
2457
 
1961
2458
  instance_type = options["machine_type"] || nil
1962
2459
  schedule = options["schedule"] || ""
@@ -1986,9 +2483,9 @@ module Cnvrg
1986
2483
  upload_output_option = "--upload_output=#{upload_output}"
1987
2484
  end
1988
2485
  options_hash = Hash[options]
1989
- options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid")
2486
+ options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid", "ignore", "data", "data_commit")
1990
2487
  exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
1991
- command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
2488
+ command = "#{exec_options} --ignore=#{ignore} #{upload_output_option} #{cmd.flatten.join(" ")}"
1992
2489
  commit_to_run = options["commit"] || nil
1993
2490
  if !schedule.nil? and !schedule.empty?
1994
2491
 
@@ -1999,7 +2496,7 @@ module Cnvrg
1999
2496
  choose_image = options["image"]
2000
2497
 
2001
2498
  if !choose_image.nil? and !choose_image.empty?
2002
- invoke :set_image,[choose_image]
2499
+ invoke :set_image, [choose_image]
2003
2500
  end
2004
2501
  image = is_project_with_docker(working_dir)
2005
2502
  if !image or !image.is_docker
@@ -2021,15 +2518,13 @@ module Cnvrg
2021
2518
  end
2022
2519
 
2023
2520
 
2024
-
2025
2521
  invoke :sync, [false], []
2026
2522
 
2027
2523
  say "Running remote experiment", Thor::Shell::Color::BLUE
2028
2524
  exp = Experiment.new(project.owner, project.slug)
2029
2525
 
2030
- res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
2526
+ res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid, path_to_cmd, data, data_commit)
2031
2527
  if Cnvrg::CLI.is_response_success(res)
2032
-
2033
2528
  # if res["result"]["machine"] == -1
2034
2529
  # say "There are no available machines", Thor::Shell::Color::BLUE
2035
2530
  # create = yes? "create new machine?", Thor::Shell::Color::YELLOW
@@ -2061,7 +2556,7 @@ module Cnvrg
2061
2556
  # end
2062
2557
  # else
2063
2558
  check = Helpers.checkmark()
2064
- str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
2559
+ str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
2065
2560
 
2066
2561
  if res["result"]["grid"]
2067
2562
  str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
@@ -2101,7 +2596,6 @@ module Cnvrg
2101
2596
  schedule = options["schedule"] || ""
2102
2597
 
2103
2598
 
2104
-
2105
2599
  if !schedule.nil? and !schedule.empty?
2106
2600
 
2107
2601
  local_timestamp = get_schedule_date
@@ -2126,7 +2620,6 @@ module Cnvrg
2126
2620
  end
2127
2621
 
2128
2622
 
2129
-
2130
2623
  invoke :sync, [false], []
2131
2624
 
2132
2625
  res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
@@ -2186,15 +2679,16 @@ module Cnvrg
2186
2679
  method_option :kernel, :type => :string, :aliases => ["--k", "-k"], :default => ""
2187
2680
  method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
2188
2681
  method_option :local, :type => :boolean, :aliases => ["-l"], :default => false
2189
- method_option :small, :type => :boolean, :aliases => ["-sm","--small"], :default => false
2190
- method_option :medium, :type => :boolean, :aliases => ["-md","--medium"], :default => false
2191
- method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
2682
+ method_option :small, :type => :boolean, :aliases => ["-sm", "--small"], :default => false
2683
+ method_option :medium, :type => :boolean, :aliases => ["-md", "--medium"], :default => false
2684
+ method_option :large, :type => :boolean, :aliases => ["-lg", "--large"], :default => false
2192
2685
  method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
2193
2686
  method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
2194
2687
  method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
2195
2688
  method_option :image, :type => :string, :aliases => ["--i"], :default => ""
2196
2689
 
2197
2690
  desc 'notebook', 'starts a notebook session remotely or locally'
2691
+
2198
2692
  def notebook
2199
2693
  local = options["local"]
2200
2694
  notebook_dir = options["notebook_dir"]
@@ -2202,14 +2696,14 @@ module Cnvrg
2202
2696
  image = options["image"]
2203
2697
 
2204
2698
  if local
2205
- invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel,:image =>image
2699
+ invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel, :image => image
2206
2700
  return
2207
2701
  else
2208
2702
  instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
2209
- "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
2703
+ "gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl" => options["gpuxxl"]}
2210
2704
  instance_type = get_instance_type(instances)
2211
2705
 
2212
- invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image =>image
2706
+ invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image => image
2213
2707
  return
2214
2708
 
2215
2709
  end
@@ -2219,7 +2713,7 @@ module Cnvrg
2219
2713
 
2220
2714
  desc 'remote_notebook', 'run notebook server on remote server'
2221
2715
  method_option :notebook_dir, :type => :string, :aliases => ["-n"], :default => "", :desc => "relative path to notebook dir from current directory"
2222
- method_option :machine_type, :type => :string, :default => ""
2716
+ method_option :machine_type, :type => :string, :default => ""
2223
2717
  method_option :kernel, :type => :string, :aliases => ["--kernel", "-k"], :default => ""
2224
2718
  method_option :image, :type => :string, :aliases => ["-i"], :default => ""
2225
2719
 
@@ -2239,7 +2733,7 @@ module Cnvrg
2239
2733
  choose_image = options["image"]
2240
2734
 
2241
2735
  if !choose_image.nil? and !choose_image.empty?
2242
- invoke :set_image,[choose_image]
2736
+ invoke :set_image, [choose_image]
2243
2737
  end
2244
2738
  @image = is_project_with_docker(working_dir)
2245
2739
  if !@image or !@image.is_docker
@@ -2250,7 +2744,7 @@ module Cnvrg
2250
2744
  end
2251
2745
  # default = yes? "use #{default_image_name} default image?", Thor::Shell::Color::YELLOW
2252
2746
  # if default
2253
- @image = Images.new(working_dir, default_image_name)
2747
+ @image = Images.new(working_dir, default_image_name)
2254
2748
  # else
2255
2749
  # exit(0)
2256
2750
  # end
@@ -2259,7 +2753,6 @@ module Cnvrg
2259
2753
  invoke :sync, [false], []
2260
2754
 
2261
2755
 
2262
-
2263
2756
  res = @image.remote_notebook(notebook_dir, instance_type, kernel)
2264
2757
  if Cnvrg::CLI.is_response_success(res)
2265
2758
  if res["result"]["machine"] == -1
@@ -2368,7 +2861,7 @@ module Cnvrg
2368
2861
  end
2369
2862
  rescue => e
2370
2863
  log_end(-1, e.message)
2371
- say "Error occurred, aborting",Thor::Shell::Color::RED
2864
+ say "Error occurred, aborting", Thor::Shell::Color::RED
2372
2865
  if container
2373
2866
  container.stop()
2374
2867
  end
@@ -2473,7 +2966,7 @@ module Cnvrg
2473
2966
  choose_image = options["image"]
2474
2967
 
2475
2968
  if !choose_image.nil? and !choose_image.empty?
2476
- invoke :set_image,[choose_image]
2969
+ invoke :set_image, [choose_image]
2477
2970
  end
2478
2971
  image = is_project_with_docker(Dir.pwd)
2479
2972
  if image and image.is_docker and !remote
@@ -2510,7 +3003,7 @@ module Cnvrg
2510
3003
  base_url = get_base_url()
2511
3004
 
2512
3005
  local_url = "/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/view/local"
2513
- command = ["/bin/bash","-lc","sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
3006
+ command = ["/bin/bash", "-lc", "sed -i 's#c.NotebookApp.base_url = .*#c.NotebookApp.base_url = \"#{local_url}\"#' /home/ds/.jupyter/jupyter_notebook_config.py"]
2514
3007
  container.exec(command, tty: true)
2515
3008
  container.stop()
2516
3009
  container.start()
@@ -2526,7 +3019,7 @@ module Cnvrg
2526
3019
  exit(1)
2527
3020
  end
2528
3021
 
2529
- url = list[list.size-1].split(" ::")
3022
+ url = list[list.size-1].split(" ::")
2530
3023
  token = url[0][(url[0].index("=")+1)..-1]
2531
3024
 
2532
3025
  # machine_activity = @note.get_machine_activity(project_dir)
@@ -2562,7 +3055,7 @@ module Cnvrg
2562
3055
  # exit(1)
2563
3056
  # end
2564
3057
  rescue => e
2565
- log_end(-1,e.message)
3058
+ log_end(-1, e.message)
2566
3059
  say "Error occurd, aborting", Thor::Shell::Color::RED
2567
3060
  if container
2568
3061
  container.stop()
@@ -2583,6 +3076,7 @@ module Cnvrg
2583
3076
  method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
2584
3077
  method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
2585
3078
  method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
3079
+
2586
3080
  def notebook_stop
2587
3081
  begin
2588
3082
  verify_logged_in(true)
@@ -2638,7 +3132,7 @@ module Cnvrg
2638
3132
  exit(0)
2639
3133
  end
2640
3134
  rescue => e
2641
- log_end(-1,e.message)
3135
+ log_end(-1, e.message)
2642
3136
  say "Error occurd, aborting"
2643
3137
  if container
2644
3138
  container.stop()
@@ -2688,7 +3182,7 @@ module Cnvrg
2688
3182
 
2689
3183
  log_end(0)
2690
3184
  rescue => e
2691
- log_end(-1,e.message)
3185
+ log_end(-1, e.message)
2692
3186
  say "Error occurd, aborting"
2693
3187
  if container
2694
3188
  container.stop()
@@ -2751,7 +3245,7 @@ module Cnvrg
2751
3245
 
2752
3246
  log_end(0)
2753
3247
  rescue => e
2754
- log_end(-1,e.message)
3248
+ log_end(-1, e.message)
2755
3249
  say "Error occurd, aborting"
2756
3250
  if container
2757
3251
  container.stop()
@@ -2766,6 +3260,7 @@ module Cnvrg
2766
3260
  end
2767
3261
 
2768
3262
  end
3263
+
2769
3264
  desc 'build', 'run commands inside containers'
2770
3265
  method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
2771
3266
 
@@ -2777,10 +3272,10 @@ module Cnvrg
2777
3272
  working_dir = is_cnvrg_dir
2778
3273
  install_file = options["install"] || nil
2779
3274
  if !install_file.nil?
2780
- commands = File.open(install_file).read.chop.gsub!("\n",",").split(",")
3275
+ commands = File.open(install_file).read.chop.gsub!("\n", ",").split(",")
2781
3276
 
2782
3277
  else
2783
- commands = [cmd.join(" ")]
3278
+ commands = [cmd.join(" ")]
2784
3279
  end
2785
3280
 
2786
3281
 
@@ -2798,25 +3293,25 @@ module Cnvrg
2798
3293
 
2799
3294
  end
2800
3295
  commands.each do |c|
2801
- if c.include? "pip"
2802
- c.sub("pip","/opt/ds/bin/pip")
2803
- end
2804
- if c.include? "pip3"
2805
- c.sub("pip3","/opt/ds3/bin/pip3")
2806
- end
3296
+ if c.include? "pip"
3297
+ c.sub("pip", "/opt/ds/bin/pip")
3298
+ end
3299
+ if c.include? "pip3"
3300
+ c.sub("pip3", "/opt/ds3/bin/pip3")
3301
+ end
2807
3302
 
2808
- say "Running #{c}", Thor::Shell::Color::BLUE
2809
- command = ["/bin/bash", "-lc", "#{c}"]
2810
- res = container.exec(command, tty: false)
2811
- if res[2] != 0
2812
- say "Could not run command: #{c}, #{res[1][0]}" , Thor::Shell::Color::RED
2813
- container.stop()
2814
- log_end(0)
3303
+ say "Running #{c}", Thor::Shell::Color::BLUE
3304
+ command = ["/bin/bash", "-lc", "#{c}"]
3305
+ res = container.exec(command, tty: false)
3306
+ if res[2] != 0
3307
+ say "Could not run command: #{c}, #{res[1][0]}", Thor::Shell::Color::RED
3308
+ container.stop()
3309
+ log_end(0)
2815
3310
 
2816
- exit(1)
2817
- end
2818
- say res[0].join("\n")
2819
- image.store_image_build_commands(working_dir,c)
3311
+ exit(1)
3312
+ end
3313
+ say res[0].join("\n")
3314
+ image.store_image_build_commands(working_dir, c)
2820
3315
  end
2821
3316
 
2822
3317
  checks = Helpers.checkmark()
@@ -2827,7 +3322,7 @@ module Cnvrg
2827
3322
 
2828
3323
  log_end(0)
2829
3324
  rescue => e
2830
- log_end(-1,e.message)
3325
+ log_end(-1, e.message)
2831
3326
  say "Error occurd, aborting", Thor::Shell::Color::RED
2832
3327
  if container
2833
3328
  container.stop()
@@ -2876,7 +3371,7 @@ module Cnvrg
2876
3371
  log_end(0)
2877
3372
  return new_image.id
2878
3373
  rescue => e
2879
- log_end(-1,e.message)
3374
+ log_end(-1, e.message)
2880
3375
  say "\nError occurred,aborting"
2881
3376
  exit(1)
2882
3377
  rescue SignalException
@@ -2939,22 +3434,22 @@ module Cnvrg
2939
3434
  end
2940
3435
  if !name.nil? and !name.empty?
2941
3436
  if name.include? " "
2942
- name.gsub!(" ","_")
3437
+ name.gsub!(" ", "_")
2943
3438
  end
2944
3439
  end
2945
- stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n",",")
3440
+ stored_commands = File.open(working_dir+"/.cnvrg/custom_image.txt").read.chop.gsub("\n", ",")
2946
3441
  if stored_commands.nil? or stored_commands.empty?
2947
3442
  say "Nothing to push", Thor::Shell::Color::BLUE
2948
3443
  exit(0)
2949
3444
  end
2950
3445
 
2951
3446
  say "Pushing new image", Thor::Shell::Color::BLUE
2952
- if image.create_custom_image(name,working_dir,stored_commands)
3447
+ if image.create_custom_image(name, working_dir, stored_commands)
2953
3448
 
2954
3449
  say "Image was updated successfully", Thor::Shell::Color::GREEN
2955
3450
  end
2956
3451
  rescue => e
2957
- log_end(-1,e.message)
3452
+ log_end(-1, e.message)
2958
3453
  say "error occurred, aborting"
2959
3454
 
2960
3455
  end
@@ -3018,7 +3513,7 @@ module Cnvrg
3018
3513
  exit(1)
3019
3514
  end
3020
3515
  rescue => e
3021
- log_end(-1,e.message)
3516
+ log_end(-1, e.message)
3022
3517
  say "Couldn't upload image file for: #{image_name}", Thor::Shell::Color::RED
3023
3518
  rescue SignalException
3024
3519
  log_end(-1)
@@ -3044,9 +3539,8 @@ module Cnvrg
3044
3539
  container.start()
3045
3540
  cnvrg_command = cmd.join(" ")
3046
3541
  command = ["/bin/bash", "-lc", "#{cnvrg_command}"]
3047
- res = container.exec(command, tty: true,wait:5400)[0]
3542
+ res = container.exec(command, tty: true, wait: 5400)[0]
3048
3543
  say res
3049
-
3050
3544
  end
3051
3545
 
3052
3546
  desc '', '', :hide => true
@@ -3054,8 +3548,13 @@ module Cnvrg
3054
3548
  def port_container(container_id)
3055
3549
  container = Docker::Container.get(container_id)
3056
3550
  say container.json["HostConfig"]["PortBindings"]["8888/tcp"][0]["HostPort"]
3551
+ end
3057
3552
 
3553
+ desc '', '', :hide => true
3058
3554
 
3555
+ def tensor_port_container(container_id)
3556
+ container = Docker::Container.get(container_id)
3557
+ say container.json["HostConfig"]["PortBindings"]["6006/tcp"][0]["HostPort"]
3059
3558
  end
3060
3559
 
3061
3560
  desc '', '', :hide => true
@@ -3071,10 +3570,9 @@ module Cnvrg
3071
3570
  method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
3072
3571
  method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
3073
3572
  method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
3074
- method_option :volume, :type => :string, :aliases => ["-v", "--v"], :default => "/home/ds/notebooks/data"
3075
3573
 
3076
3574
 
3077
- def config_remote(image_name, port=7654,tensport=6006)
3575
+ def config_remote(image_name, port=7654, tensport=6006)
3078
3576
  local_images = Docker::Image.all
3079
3577
 
3080
3578
  docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -3097,7 +3595,6 @@ module Cnvrg
3097
3595
  'ExposedPorts' => {
3098
3596
  '8888/tcp' => {},
3099
3597
  },
3100
- 'VolumesFrom' =>volume_from,
3101
3598
  'HostConfig' => {
3102
3599
  'PortBindings' => {
3103
3600
  '8888/tcp' => [
@@ -3124,7 +3621,7 @@ module Cnvrg
3124
3621
  say "#{container.id}:#{port}##{tensport}"
3125
3622
  rescue => e
3126
3623
  if e.message.include? "is not running"
3127
- return config_remote(image_name, port-1,tensport-1)
3624
+ return config_remote(image_name, port-1, tensport-1)
3128
3625
  end
3129
3626
 
3130
3627
  if container
@@ -3133,13 +3630,14 @@ module Cnvrg
3133
3630
  return false
3134
3631
  end
3135
3632
  end
3633
+
3136
3634
  desc '', '', :hide => true
3137
3635
  method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
3138
3636
  method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
3139
3637
  method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
3140
3638
 
3141
3639
 
3142
- def config_remote_gpu(image_name, port=7654,tensport=6006)
3640
+ def config_remote_gpu(image_name, port=7654, tensport=6006)
3143
3641
  local_images = Docker::Image.all
3144
3642
 
3145
3643
  docker_image_local = local_images.map { |x| x.info["RepoTags"] }.flatten.select { |y| y.eql? "#{image_name}:latest" }.flatten
@@ -3174,7 +3672,7 @@ module Cnvrg
3174
3672
  # }
3175
3673
 
3176
3674
  container_id = `nvidia-docker run -itd -p #{port}:8888 -p #{tensport}:6006 -w #{app_dir} #{image_name}:latest #{cmd} `
3177
- container_id = container_id.gsub("\n","")
3675
+ container_id = container_id.gsub("\n", "")
3178
3676
  container = Docker::Container.get(container_id)
3179
3677
  # container.start()
3180
3678
  command = ["/bin/bash", "-lc", "sudo echo -e \"#{login_content}\" >/home/ds/.netrc"]
@@ -3191,7 +3689,7 @@ module Cnvrg
3191
3689
  rescue => e
3192
3690
  if e.message.include? "is not running"
3193
3691
  puts "running asgain with: #{port-1} #{tensport-1}"
3194
- return config_remote_gpu(image_name, port-1,tensport-1)
3692
+ return config_remote_gpu(image_name, port-1, tensport-1)
3195
3693
  end
3196
3694
 
3197
3695
  if container
@@ -3344,7 +3842,6 @@ module Cnvrg
3344
3842
  end
3345
3843
 
3346
3844
 
3347
-
3348
3845
  return dir_path
3349
3846
 
3350
3847
  else
@@ -3378,13 +3875,13 @@ module Cnvrg
3378
3875
  owner = Cnvrg::CLI.get_owner()
3379
3876
  res = Cnvrg::API.request("users/#{owner}/images/list", 'GET')
3380
3877
  if Cnvrg::CLI.is_response_success(res)
3381
- printf "%-20s %-20s %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
3878
+ printf "%-20s %-20s %-30s %-20s %-20s\n", "name", "project", "created by", "is_public", "last updated"
3382
3879
  res["result"]["images"].each do |u|
3383
3880
  time = Time.parse(u["created_at"])
3384
3881
  update_at = get_local_time(time)
3385
3882
  created_by = u["created_by"]
3386
3883
 
3387
- printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
3884
+ printf "%-20s %-20s %-30s %-20s %-20s\n", u["name"], u["project"], created_by, u["is_public"], update_at
3388
3885
  end
3389
3886
  end
3390
3887
  log_end(0)
@@ -3728,12 +4225,13 @@ module Cnvrg
3728
4225
  return owner
3729
4226
  end
3730
4227
  end
4228
+
3731
4229
  def get_base_url
3732
4230
  home_dir = File.expand_path('~')
3733
4231
 
3734
4232
  config = YAML.load_file(home_dir+ "/.cnvrg/config.yml")
3735
4233
  api = config.to_h[:api]
3736
- return api.gsub!("/api","")
4234
+ return api.gsub!("/api", "")
3737
4235
  end
3738
4236
 
3739
4237
  def get_project_home
@@ -3857,6 +4355,18 @@ module Cnvrg
3857
4355
  return false
3858
4356
  end
3859
4357
  end
4358
+ def data_dir_include()
4359
+ all_dirs = Dir.glob("**/*/", File::FNM_DOTMATCH)
4360
+ all_dirs.flatten!
4361
+ all_dirs.each do |a|
4362
+ if a.include? "/.cnvrg"
4363
+ ignore = File.dirname(a)
4364
+ return ignore
4365
+ end
4366
+ end
4367
+ return nil
4368
+
4369
+ end
3860
4370
 
3861
4371
  def verify_software_installed(software)
3862
4372
  begin
@@ -4032,20 +4542,21 @@ module Cnvrg
4032
4542
  return new_time
4033
4543
 
4034
4544
  end
4545
+
4035
4546
  def update_deleted(deleted)
4036
4547
  final = []
4037
4548
  deleted.each do |d|
4038
- all_subs = d.split("/")
4039
- to_add = true
4040
- value = all_subs[0]
4041
- all_subs.each_with_index do |a,i|
4042
- if final.include? value+"/"
4043
- to_add = false
4044
- break
4045
- end
4046
- value = value+"/"+all_subs[i+1] if i <all_subs.size-1
4549
+ all_subs = d.split("/")
4550
+ to_add = true
4551
+ value = all_subs[0]
4552
+ all_subs.each_with_index do |a, i|
4553
+ if final.include? value+"/"
4554
+ to_add = false
4555
+ break
4047
4556
  end
4048
- final << d if to_add
4557
+ value = value+"/"+all_subs[i+1] if i <all_subs.size-1
4558
+ end
4559
+ final << d if to_add
4049
4560
 
4050
4561
 
4051
4562
  end
@@ -4053,7 +4564,8 @@ module Cnvrg
4053
4564
 
4054
4565
  return final
4055
4566
  end
4056
- def get_cmd_path_in_dir(main_dir,sub_dir)
4567
+
4568
+ def get_cmd_path_in_dir(main_dir, sub_dir)
4057
4569
  first = Pathname.new main_dir
4058
4570
  second = Pathname.new sub_dir
4059
4571
  relative = second.relative_path_from first
@@ -4064,10 +4576,18 @@ module Cnvrg
4064
4576
  end
4065
4577
  end
4066
4578
 
4579
+ def create_tar(path_in, path_out, tar_files)
4580
+ `tar -czf #{path_out} -T #{tar_files} > /dev/null 2>&1`
4581
+ return $?.success?
4582
+ end
4067
4583
 
4068
-
4584
+ def extarct_tar(file_path, dir_path)
4585
+ `tar -xvf #{file_path} -C #{dir_path} > /dev/null 2>&1`
4586
+ return $?.success?
4587
+ end
4069
4588
  end
4070
4589
  end
4071
4590
  end
4072
4591
 
4073
4592
 
4593
+