cnvrg 0.0.145 → 0.0.146

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b633d9c59001fa185be3d8c960e812a421ecef4e
4
- data.tar.gz: bb5b2d41f8d4c4f8a95ce30334cc4e034c2e9bc4
3
+ metadata.gz: db813c4622a353a9e9d60678849c904d4268d28f
4
+ data.tar.gz: ecbc4ec2f9acc052ba1b61b18a88f14c15eb0312
5
5
  SHA512:
6
- metadata.gz: 200acbdc094104d6666b5ad5b1715dd9d241e2c2cd7e3d70d77da0f6e672feb57305d54bd684e721141141201289b3b20aa993bab1bc04d43ea8465dac41eb8c
7
- data.tar.gz: 4a79957cb3800306ca5e15cd849ebb02afefd11913eca6f9f7b0145d8133f635c111e6363fdbe0390241405874fc4465c8744afc704dd822517742502ecb8408
6
+ metadata.gz: fcaf8b6352e76e300d75ea4db289b3c58c8cfe785522baa100cafa271afea54e0987dea60ab79387e5552258f55209cb667cff2f840d61199d28b5a44c4d8e92
7
+ data.tar.gz: 42b8ba42106b5d6321c56cc94db3e434102f6323941a3084cbf8a5bbef9242f154fca542591c3fce1a83f568893f14142b61392b7848120d50c13299476f35a9
@@ -18,6 +18,8 @@ require 'cnvrg/project'
18
18
  require 'cnvrg/files'
19
19
  require 'cnvrg/experiment'
20
20
  require 'cnvrg/Images'
21
+ require 'cnvrg/dataset'
22
+ require 'cnvrg/datafiles'
21
23
  require 'etc'
22
24
  require 'logstash-logger'
23
25
  require 'cnvrg/job'
@@ -29,8 +31,12 @@ require 'fileutils'
29
31
  require 'zip'
30
32
  require 'active_support/all'
31
33
  require 'thor'
34
+ require 'pathname'
35
+ require 'cnvrg/data'
36
+
32
37
 
33
38
  # DEV VERSION
39
+
34
40
  #
35
41
  module Cnvrg
36
42
  class CLI < Thor
@@ -38,18 +44,6 @@ module Cnvrg
38
44
  INSTALLATION_URLS = {docker: "https://docs.docker.com/engine/installation/", jupyter: "http://jupyter.readthedocs.io/en/latest/install.html"}
39
45
  IP="localhost"
40
46
  PORT=7654
41
- desc '', ''
42
-
43
- def printable_commands(all = true, subcommand = false)
44
- (all ? all_commands : commands).map do |_, command|
45
- next if command.hidden? or (command.description.empty? and command.usage.empty?)
46
- item = []
47
- item << banner(command, false, subcommand)
48
- item << (command.description ? "# #{command.description.gsub(/\s+/m, ' ')}" : "")
49
- item
50
- end.compact
51
- end
52
-
53
47
  class << self
54
48
  # Hackery.Take the run method away from Thor so that we can redefine it.
55
49
  def is_thor_reserved_word?(word, type)
@@ -57,11 +51,11 @@ module Cnvrg
57
51
  super
58
52
  end
59
53
  end
54
+ desc "data", "upload and manage datasets", :hide =>true
55
+ subcommand "data", Data
60
56
 
61
- desc "", ""
62
- method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
63
-
64
- def testt
57
+ desc "", "" , :hide => true
58
+ def test
65
59
  # image_settings = {
66
60
  # 'Image' => "cnvrg:latest",
67
61
  # 'User' => 'ds',
@@ -86,7 +80,6 @@ module Cnvrg
86
80
 
87
81
 
88
82
  desc 'version', 'Prints cnvrg current version'
89
-
90
83
  def version
91
84
  puts Cnvrg::VERSION
92
85
 
@@ -95,7 +88,6 @@ module Cnvrg
95
88
  map %w(-v --version) => :version
96
89
 
97
90
  desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
98
-
99
91
  def set_api_url(url)
100
92
  home_dir = File.expand_path('~')
101
93
  if !url.end_with? "/api"
@@ -136,7 +128,7 @@ module Cnvrg
136
128
  end
137
129
  end
138
130
 
139
- desc '', ''
131
+ desc '', '', :hide => true
140
132
 
141
133
  def set_remote_api_url(owner, current_user, url)
142
134
  home_dir = File.expand_path('~')
@@ -181,27 +173,41 @@ module Cnvrg
181
173
  owners = result["owners"]
182
174
  urls = result["urls"]
183
175
  choose_owner = result["username"]
184
-
185
176
  if owners.empty?
186
177
  else
187
178
  owners << choose_owner
188
179
  chosen = false
189
180
  while !chosen
190
- choose_owner = ask("Choose default owner:\n"+owners.join("\n")+"\n")
181
+ owners_id = owners.each_with_index.map{|x,i| "#{i+1}. #{x}"}
182
+ choose_owner = ask("Choose default owner:\n"+owners_id.join("\n")+"\n")
183
+
184
+ if choose_owner =~ /[[:digit:]]/
185
+ ow_index = choose_owner.to_i-1
186
+ if ow_index<0 or ow_index >= owners.size
187
+ say "No such owner, please choose again", Thor::Shell::Color::BLUE
188
+ chosen = false
189
+ next
190
+ end
191
+ choose_owner = owners[choose_owner.to_i-1]
192
+ chosen = true
191
193
 
192
- owners_lower = owners.map { |o| o.downcase }
193
- ow_index = owners_lower.index(choose_owner.downcase)
194
- if ow_index.nil?
195
- say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
196
194
  else
197
- chosen = true
195
+
196
+ owners_lower = owners.map { |o| o.downcase }
197
+ ow_index = owners_lower.index(choose_owner.downcase)
198
+ if ow_index.nil?
199
+ say "Could not find owner named #{choose_owner}", Thor::Shell::Color::RED
200
+ else
201
+ chosen = true
202
+ end
198
203
  end
204
+
199
205
  end
200
206
 
201
207
 
202
208
  end
203
- if set_owner(owner, username,urls[ow_index])
204
- say "Setting default owner: #{owner}", Thor::Shell::Color::GREEN
209
+ if set_owner(choose_owner, result["username"], urls[ow_index])
210
+ say "Setting default owner: #{choose_owner}", Thor::Shell::Color::GREEN
205
211
  else
206
212
  say "Setting default owenr has failed, try to run cnvrg --config-default-owner", Thor::Shell::Color::RED
207
213
  end
@@ -343,8 +349,8 @@ module Cnvrg
343
349
 
344
350
  ## Projects
345
351
  desc 'new', 'Create a new cnvrg project'
346
- method_option :clean, :type => :boolean, :aliases => ["-c", "--c"], :default => false
347
- method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
352
+ method_option :clean, :type => :boolean, :aliases => ["-c"], :default => false
353
+ method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
348
354
 
349
355
  def new(project_name)
350
356
  begin
@@ -468,8 +474,8 @@ module Cnvrg
468
474
 
469
475
  end
470
476
  desc 'link', 'Link current directory to a new cnvrg project'
471
- method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
472
- method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
477
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
478
+ method_option :docker_image, :type => :string, :aliases => ["-d"], :default => ""
473
479
 
474
480
  def link
475
481
  begin
@@ -550,6 +556,252 @@ module Cnvrg
550
556
  exit(1)
551
557
  end
552
558
  end
559
+
560
+ desc 'init_data', 'Init dataset directory', :hide =>true
561
+ method_option :public, :type => :boolean, :aliases => ["-p", "--public"], :default => false
562
+
563
+ def init_data
564
+ begin
565
+ verify_logged_in(false)
566
+ log_start(__method__, args, options)
567
+ dataset_name =File.basename(Dir.getwd)
568
+ if File.directory?(Dir.getwd+"/.cnvrg")
569
+ config = YAML.load_file("#{Dir.getwd}/.cnvrg/config.yml")
570
+ say "Directory is already linked to #{config[:dataset_slug]}", Thor::Shell::Color::RED
571
+
572
+ exit(0)
573
+ end
574
+ say "Init dataset: #{dataset_name}", Thor::Shell::Color::BLUE
575
+
576
+ working_dir = Dir.getwd
577
+ owner = CLI.get_owner
578
+ if Dataset.init(owner,dataset_name, options["public"])
579
+ path = Dir.pwd
580
+ @dataset = Dataset.new(path)
581
+ @dataset.generate_idx()
582
+
583
+ url = @dataset.url
584
+ check = Helpers.checkmark
585
+ say "#{check} Link finished successfully", Thor::Shell::Color::GREEN
586
+ say "#{dataset_name}'s location is: #{url}\n", Thor::Shell::Color::GREEN
587
+ log_end(0)
588
+
589
+ else
590
+ log_end(1, "can't create dataset")
591
+ @dataset.revert(working_dir) unless @dataset.nil?
592
+ say "Error creating dataset, please contact support.", Thor::Shell::Color::RED
593
+ exit(0)
594
+ end
595
+ rescue SignalException
596
+ log_end(-1)
597
+
598
+ say "\nAborting"
599
+ exit(1)
600
+ end
601
+ end
602
+ desc 'upload_data', 'Upload data files', :hide=>true
603
+ method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
604
+ method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
605
+ method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
606
+ method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
607
+
608
+ def upload_data(sync=false,direct=false)
609
+
610
+ begin
611
+ verify_logged_in(true)
612
+ log_start(__method__, args, options)
613
+ dataset_dir = is_cnvrg_dir(Dir.pwd)
614
+ @dataset = Dataset.new(dataset_dir)
615
+
616
+ @files = Cnvrg::Datafiles.new(@dataset.owner, @dataset.slug)
617
+ ignore = options[:ignore] || []
618
+ if !@dataset.update_ignore_list(ignore)
619
+ say "Couldn't append new ignore files to .cnvrgignore", Thor::Shell::Color::YELLOW
620
+ end
621
+ result = @dataset.compare_idx(false)
622
+ commit = result["result"]["commit"]
623
+ if commit != @dataset.last_local_commit and !@dataset.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
624
+ log_end(0)
625
+
626
+ say "Remote server has an updated version, please run `cnvrg download` first, or alternatively: `cnvrg sync`", Thor::Shell::Color::YELLOW
627
+ exit(1)
628
+ end
629
+
630
+ say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
631
+ result = result["result"]["tree"]
632
+ # if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
633
+ # all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
634
+ # if all.size == 1
635
+ # num = "conflict"
636
+ # else
637
+ # num = "conflicts"
638
+ # end
639
+ # say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
640
+ # say "#{all.join("\n")}"
641
+ # say "Please fix #{num}, and retry", Thor::Shell::Color::RED
642
+ # exit(1)
643
+ #
644
+ # end
645
+ check = Helpers.checkmark()
646
+
647
+ if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
648
+ log_end(0)
649
+ say "#{check} Dataset is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
650
+ return true
651
+ end
652
+ update_count = 0
653
+ update_total = result["added"].size + result["updated_on_local"].size + result["deleted"].size
654
+ successful_updates = []
655
+ successful_deletions = []
656
+ if options["verbose"]
657
+ if update_total == 1
658
+ say "Updating #{update_total} file", Thor::Shell::Color::BLUE
659
+ else
660
+ say "Updating #{update_total} files", Thor::Shell::Color::BLUE
661
+ end
662
+ else
663
+ say "Syncing files", Thor::Shell::Color::BLUE unless (options["sync"] or sync)
664
+
665
+ end
666
+
667
+ # Start commit
668
+
669
+ commit_sha1 = @files.start_commit(false)["result"]["commit_sha1"]
670
+
671
+ # upload / update
672
+ begin
673
+ (result["added"] + result["updated_on_local"]).each do |f|
674
+ absolute_path = "#{@dataset.local_path}/#{f}"
675
+ relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
676
+ if File.directory?(absolute_path)
677
+ resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
678
+ if resDir
679
+ update_count += 1
680
+ successful_updates<< relative_path
681
+ end
682
+ else
683
+ res = @files.upload_file(absolute_path, relative_path, commit_sha1)
684
+ if res
685
+ update_count += 1
686
+ successful_updates<< relative_path
687
+ else
688
+ @files.rollback_commit(commit_sha1)
689
+ log_end(1, "can't upload, Rolling Back all changes")
690
+ say "Couldn't upload, Rolling Back all changes.", Thor::Shell::Color::RED
691
+ exit(0)
692
+ end
693
+ end
694
+ end
695
+
696
+ # delete
697
+ deleted = update_deleted(result["deleted"])
698
+ deleted.each do |f|
699
+ relative_path = f.gsub(/^#{@dataset.local_path + "/"}/, "")
700
+ if relative_path.end_with?("/")
701
+ if @files.delete_dir(f, relative_path, commit_sha1)
702
+ # update_count += 1
703
+ successful_updates<< relative_path
704
+ end
705
+ else
706
+ if @files.delete_file(f, relative_path, commit_sha1)
707
+ # update_count += 1
708
+ successful_updates<< relative_path
709
+ end
710
+ end
711
+ end
712
+
713
+ rescue SignalException
714
+ log_end(-1)
715
+ @files.rollback_commit(commit_sha1)
716
+ say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
717
+ exit(0)
718
+ rescue => e
719
+ log_end(-1, e.message)
720
+ @files.rollback_commit(commit_sha1)
721
+ say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
722
+ exit(0)
723
+ end
724
+ if !result["deleted"].nil? and !result["deleted"].empty?
725
+ update_count += result["deleted"].size
726
+ end
727
+ if update_count == update_total
728
+ res = @files.end_commit(commit_sha1)
729
+ if (Cnvrg::CLI.is_response_success(res, false))
730
+ # save idx
731
+ begin
732
+ @dataset.update_idx_with_files_commits!((successful_deletions+successful_updates), res["result"]["commit_time"])
733
+
734
+ @dataset.update_idx_with_commit!(commit_sha1)
735
+ rescue => e
736
+ log_end(-1, e.message)
737
+ @files.rollback_commit(commit_sha1)
738
+ say "Couldn't commit updates, Rolling Back all changes.", Thor::Shell::Color::RED
739
+ exit(1)
740
+
741
+ end
742
+ if options["verbose"]
743
+ say "#{check} Done", Thor::Shell::Color::BLUE
744
+ if successful_updates.size >0
745
+ say "Updated:", Thor::Shell::Color::GREEN
746
+ suc = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
747
+ say suc.join("\n"), Thor::Shell::Color::GREEN
748
+ end
749
+ if successful_deletions.size >0
750
+ say "Deleted:", Thor::Shell::Color::GREEN
751
+ del = successful_updates.map { |x| x=Helpers.checkmark() +" "+x }
752
+ say del.join("\n"), Thor::Shell::Color::GREEN
753
+ end
754
+ say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
755
+ else
756
+ if (options["sync"] or sync) and direct
757
+ say "#{check} Syncing dataset completed successfully", Thor::Shell::Color::GREEN
758
+
759
+ else
760
+ say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
761
+
762
+ end
763
+
764
+ end
765
+
766
+ log_end(0)
767
+ else
768
+ @files.rollback_commit(commit_sha1)
769
+ log_end(1, "error. Rolling Back all changes")
770
+ say "Error. Rolling Back all changes.", Thor::Shell::Color::RED
771
+ end
772
+ else
773
+ log_end(1, "error. Rolling Back all changes")
774
+ say "Error occurd, \nRolling back", Thor::Shell::Color::RED
775
+
776
+ @files.rollback_commit(commit_sha1)
777
+ end
778
+ rescue =>e
779
+
780
+ log_end(-1)
781
+
782
+ say "Error occurd, \nAborting", Thor::Shell::Color::RED
783
+ @files.rollback_commit(commit_sha1)
784
+ exit(1)
785
+ rescue SignalException
786
+ log_end(-1)
787
+
788
+ say "\nAborting", Thor::Shell::Color::BLUE
789
+ say "\nRolling back all changes", Thor::Shell::Color::BLUE
790
+ @files.rollback_commit(commit_sha1)
791
+ exit(1)
792
+ end
793
+
794
+ end
795
+ desc 'unlink','Unlink a project from current directory'
796
+ def create_volume
797
+ verify_logged_in(true)
798
+ log_start(__method__, args, options)
799
+ dataset_dir = is_cnvrg_dir(Dir.pwd)
800
+ @dataset = Dataset.new(dataset_dir)
801
+ @dataset.create_volume()
802
+
803
+ end
804
+
553
805
  desc 'unlink','Unlink a project from current directory'
554
806
  def unlink
555
807
  verify_logged_in(false)
@@ -733,7 +985,7 @@ module Cnvrg
733
985
  exit(1)
734
986
  end
735
987
  end
736
- desc '', 's'
988
+ desc '', '' , :hide => true
737
989
  def revert_exp
738
990
  begin
739
991
  log_start(__method__, args, options)
@@ -754,12 +1006,12 @@ module Cnvrg
754
1006
 
755
1007
 
756
1008
  desc 'upload', 'Upload updated files'
757
- method_option :ignore, :type => :array, :aliases => ["-i", "--i"], :desc => "ignore following files"
758
- method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
759
- method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
760
- method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
1009
+ method_option :ignore, :type => :array, :aliases => ["-i"], :desc => "ignore following files"
1010
+ method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1011
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1012
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
761
1013
 
762
- def upload(link=false, sync=false)
1014
+ def upload(link=false, sync=false,direct=false)
763
1015
 
764
1016
  begin
765
1017
  verify_logged_in(true)
@@ -807,9 +1059,11 @@ module Cnvrg
807
1059
  # exit(1)
808
1060
  #
809
1061
  # end
1062
+ check = Helpers.checkmark()
1063
+
810
1064
  if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
811
1065
  log_end(0)
812
- say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
1066
+ say "#{check} Project is up to date", Thor::Shell::Color::GREEN unless ((options["sync"] or sync) and !direct)
813
1067
  return true
814
1068
  end
815
1069
  update_count = 0
@@ -906,7 +1160,6 @@ module Cnvrg
906
1160
  if image and image.is_docker
907
1161
  image.update_image_activity(commit_sha1, nil)
908
1162
  end
909
- check = Helpers.checkmark()
910
1163
 
911
1164
  if options["verbose"]
912
1165
  say "#{check} Done", Thor::Shell::Color::BLUE
@@ -922,10 +1175,10 @@ module Cnvrg
922
1175
  end
923
1176
  say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
924
1177
  else
925
- if !options["sync"] and !sync
926
- # say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
1178
+ if (options["sync"] or sync) and direct
1179
+ say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
927
1180
 
928
- # else
1181
+ else
929
1182
  say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
930
1183
 
931
1184
  end
@@ -940,7 +1193,7 @@ module Cnvrg
940
1193
  end
941
1194
  else
942
1195
  log_end(1, "error. Rolling Back all changes")
943
- say "Error occurd, \nRolling back", Thor::Shell::Color::BLUE
1196
+ say "Error occurd, \nRolling back", Thor::Shell::Color::RED
944
1197
 
945
1198
  @files.rollback_commit(commit_sha1)
946
1199
  end
@@ -948,7 +1201,7 @@ module Cnvrg
948
1201
 
949
1202
  log_end(-1)
950
1203
 
951
- say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
1204
+ say "Error occurd, \nAborting", Thor::Shell::Color::RED
952
1205
  @files.rollback_commit(commit_sha1)
953
1206
  exit(1)
954
1207
  rescue SignalException
@@ -963,9 +1216,9 @@ module Cnvrg
963
1216
  end
964
1217
 
965
1218
  desc 'download', 'Download updated files'
966
- method_option :new_branch, :type => :boolean, :aliases => ["-nb", "--nb"], :desc => "create new branch of commits"
967
- method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
968
- method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
1219
+ method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1220
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1221
+ method_option :sync, :type => :boolean, :aliases => ["-s"], :default => false
969
1222
 
970
1223
  def download(sync=false)
971
1224
  begin
@@ -1050,7 +1303,7 @@ module Cnvrg
1050
1303
  say successful_changes.join("\n"), Thor::Shell::Color::GREEN
1051
1304
  say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
1052
1305
  else
1053
- say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN if !sync
1306
+ say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN unless (sync or options["sync"])
1054
1307
  end
1055
1308
 
1056
1309
 
@@ -1163,10 +1416,11 @@ module Cnvrg
1163
1416
 
1164
1417
 
1165
1418
  desc 'sync', 'Sync with remote server'
1166
- method_option :new_branch, :type => :boolean, :aliases => ["--nb"], :desc => "create new branch of commits"
1167
- method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
1419
+ method_option :new_branch, :type => :boolean, :aliases => ["-nb"], :desc => "create new branch of commits"
1420
+ method_option :verbose, :type => :boolean, :aliases => ["-v"], :default => false
1168
1421
 
1169
- def sync
1422
+ def sync(direct=true)
1423
+ verify_logged_in(true) if direct
1170
1424
  if options["verbose"]
1171
1425
  say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
1172
1426
  else
@@ -1175,7 +1429,7 @@ module Cnvrg
1175
1429
 
1176
1430
 
1177
1431
  invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
1178
- invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
1432
+ invoke :upload, [link=false, sync=true,direct=direct], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
1179
1433
 
1180
1434
 
1181
1435
  end
@@ -1227,7 +1481,7 @@ module Cnvrg
1227
1481
  end
1228
1482
 
1229
1483
 
1230
- desc '', ''
1484
+ desc '', '' , :hide => true
1231
1485
  method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
1232
1486
  method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
1233
1487
  method_option :title, :type => :string, :aliases => ["-t"], :default => ""
@@ -1244,7 +1498,8 @@ module Cnvrg
1244
1498
  memory_average = 0
1245
1499
  verify_logged_in(true)
1246
1500
  log_start(__method__, args, options)
1247
- working_dir = Dir.pwd
1501
+ working_dir = is_cnvrg_dir
1502
+ script_path = get_cmd_path_in_dir(working_dir,Dir.pwd)
1248
1503
 
1249
1504
 
1250
1505
  sync_before = options["sync_before"]
@@ -1268,7 +1523,7 @@ module Cnvrg
1268
1523
  else
1269
1524
  if sync_before
1270
1525
  # Sync before run
1271
- invoke :sync, [], :new_branch => is_new_branch
1526
+ invoke :sync, [false], :new_branch => is_new_branch
1272
1527
  end
1273
1528
  end
1274
1529
  #set image for the project
@@ -1308,8 +1563,8 @@ module Cnvrg
1308
1563
  platform = RUBY_PLATFORM
1309
1564
  machine_name = Socket.gethostname
1310
1565
  begin
1311
- machine_activity = @exp.get_machine_activity(Dir.pwd)
1312
- @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity)
1566
+ machine_activity = @exp.get_machine_activity(working_dir)
1567
+ @exp.start(cmd, platform, machine_name, start_commit, title, email_notification, machine_activity,script_path)
1313
1568
  unless @exp.slug.nil?
1314
1569
  real = Time.now
1315
1570
  exp_success = true
@@ -1418,8 +1673,9 @@ module Cnvrg
1418
1673
  if sync_after
1419
1674
  # Sync after run
1420
1675
 
1676
+
1421
1677
  download(sync=true)
1422
- upload(link=false, sync=true)
1678
+ upload(link=false, sync=true,direct=false)
1423
1679
 
1424
1680
  end
1425
1681
  end_commit = @project.last_local_commit
@@ -1457,7 +1713,7 @@ module Cnvrg
1457
1713
  end
1458
1714
  end
1459
1715
 
1460
- desc '', ''
1716
+ desc '', '', :hide => true
1461
1717
  method_option :sync_before, :type => :boolean, :aliases => ["-sb", "--sb"], :default => true
1462
1718
  method_option :sync_after, :type => :boolean, :aliases => ["-sa", "--sa"], :default => true
1463
1719
  method_option :title, :type => :string, :aliases => ["-t", "--t"], :default => ""
@@ -1642,7 +1898,7 @@ module Cnvrg
1642
1898
  end
1643
1899
  end
1644
1900
 
1645
- desc '', ''
1901
+ desc '', '', :hide => true
1646
1902
  method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
1647
1903
  method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
1648
1904
  method_option :title, :type => :string, :aliases => ["-t"], :default => ""
@@ -1659,6 +1915,8 @@ module Cnvrg
1659
1915
  verify_logged_in(true)
1660
1916
  log_start(__method__, args, options)
1661
1917
  working_dir = is_cnvrg_dir
1918
+ path_to_cmd = get_cmd_path_in_dir(working_dir,Dir.pwd)
1919
+
1662
1920
  begin
1663
1921
  grid = options["grid"] || nil
1664
1922
 
@@ -1726,12 +1984,12 @@ module Cnvrg
1726
1984
 
1727
1985
 
1728
1986
 
1729
- invoke :sync, [], []
1987
+ invoke :sync, [false], []
1730
1988
 
1731
1989
  say "Running remote experiment", Thor::Shell::Color::BLUE
1732
1990
  exp = Experiment.new(project.owner, project.slug)
1733
1991
 
1734
- res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid)
1992
+ res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid,path_to_cmd)
1735
1993
  if Cnvrg::CLI.is_response_success(res)
1736
1994
 
1737
1995
  # if res["result"]["machine"] == -1
@@ -1831,7 +2089,7 @@ module Cnvrg
1831
2089
 
1832
2090
 
1833
2091
 
1834
- invoke :sync, [], []
2092
+ invoke :sync, [false], []
1835
2093
 
1836
2094
  res = project.deploy(file_to_run, function, nil, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
1837
2095
 
@@ -1898,9 +2156,8 @@ module Cnvrg
1898
2156
  method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
1899
2157
  method_option :image, :type => :string, :aliases => ["--i"], :default => ""
1900
2158
 
1901
- desc 'starts a notebook session', 'starts a notebook session remotely or locally'
1902
-
1903
- def notebook()
2159
+ desc 'notebook', 'starts a notebook session remotely or locally'
2160
+ def notebook
1904
2161
  local = options["local"]
1905
2162
  notebook_dir = options["notebook_dir"]
1906
2163
  kernel = options["kernel"]
@@ -1961,7 +2218,7 @@ module Cnvrg
1961
2218
  # end
1962
2219
  end
1963
2220
 
1964
- invoke :sync, [], []
2221
+ invoke :sync, [false], []
1965
2222
 
1966
2223
 
1967
2224
 
@@ -2004,7 +2261,7 @@ module Cnvrg
2004
2261
  note_url = res["result"]["notebook_url"]
2005
2262
  @image.set_note_url(note_url)
2006
2263
  check = Helpers.checkmark()
2007
- say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
2264
+ say "#{check} Notebook is on: #{Cnvrg::Helpers.remote_url}/#{@image.owner}/projects/#{@image.project_slug}/notebook_sessions/show/#{note_url}", Thor::Shell::Color::GREEN
2008
2265
  # Launchy.open(url)
2009
2266
 
2010
2267
  exit(0)
@@ -2209,7 +2466,7 @@ module Cnvrg
2209
2466
  end
2210
2467
 
2211
2468
  end
2212
- invoke :sync, [], :verbose => options["verbose"]
2469
+ invoke :sync, [false], :verbose => options["verbose"]
2213
2470
  say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
2214
2471
  #replace url
2215
2472
  base_url = get_base_url()
@@ -2288,7 +2545,6 @@ module Cnvrg
2288
2545
  method_option :notebook_dir, :type => :string, :aliases => ["-n", "--n"], :default => "", :desc => "relative path to notebook dir from current directory"
2289
2546
  method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false, :desc => "run on remote machine"
2290
2547
  method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
2291
-
2292
2548
  def notebook_stop
2293
2549
  begin
2294
2550
  verify_logged_in(true)
@@ -2299,7 +2555,7 @@ module Cnvrg
2299
2555
 
2300
2556
 
2301
2557
  say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE if options["verbose"]
2302
- invoke :sync, [], :verbose => options["verbose"]
2558
+ invoke :sync, [false], :verbose => options["verbose"]
2303
2559
 
2304
2560
  say "Done Syncing", Thor::Shell::Color::BLUE if options["verbose"]
2305
2561
 
@@ -2472,7 +2728,7 @@ module Cnvrg
2472
2728
  end
2473
2729
 
2474
2730
  end
2475
- desc 'run commands inside containers', 'run commands inside containers'
2731
+ desc 'build', 'run commands inside containers'
2476
2732
  method_option :install, :type => :string, :aliases => ["--i"], :default => nil, :desc => "Install from the given instructions file"
2477
2733
 
2478
2734
  def build(*cmd)
@@ -2627,7 +2883,7 @@ module Cnvrg
2627
2883
 
2628
2884
  end
2629
2885
 
2630
- desc 'push image to cnvrg repository', 'push image to cnvrg repository'
2886
+ desc 'push', 'push image to cnvrg repository'
2631
2887
 
2632
2888
  def push(*name)
2633
2889
  verify_logged_in(true)
@@ -2734,7 +2990,7 @@ module Cnvrg
2734
2990
  end
2735
2991
  end
2736
2992
 
2737
- desc '', ''
2993
+ desc '', '', :hide => true
2738
2994
 
2739
2995
  def upload_log()
2740
2996
  log_path = '/home/ds/app/uwsgi.log'
@@ -2743,7 +2999,7 @@ module Cnvrg
2743
2999
 
2744
3000
  end
2745
3001
 
2746
- desc '', ''
3002
+ desc '', '', :hide => true
2747
3003
 
2748
3004
  def exec_container(container_id, *cmd)
2749
3005
  container = Docker::Container.get(container_id)
@@ -2755,7 +3011,7 @@ module Cnvrg
2755
3011
 
2756
3012
  end
2757
3013
 
2758
- desc '', ''
3014
+ desc '', '', :hide => true
2759
3015
 
2760
3016
  def port_container(container_id)
2761
3017
  container = Docker::Container.get(container_id)
@@ -2764,7 +3020,7 @@ module Cnvrg
2764
3020
 
2765
3021
  end
2766
3022
 
2767
- desc '', ''
3023
+ desc '', '', :hide => true
2768
3024
 
2769
3025
  def stop_container(container_id)
2770
3026
  container = Docker::Container.get(container_id)
@@ -2773,7 +3029,7 @@ module Cnvrg
2773
3029
 
2774
3030
  end
2775
3031
 
2776
- desc '', ''
3032
+ desc '', '', :hide => true
2777
3033
  method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
2778
3034
  method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
2779
3035
  method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
@@ -2836,7 +3092,7 @@ module Cnvrg
2836
3092
  return false
2837
3093
  end
2838
3094
  end
2839
- desc '', ''
3095
+ desc '', '', :hide => true
2840
3096
  method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
2841
3097
  method_option :app_dir, :type => :string, :aliases => ["-d", "--d"], :default => "/home/ds/notebooks"
2842
3098
  method_option :cmd, :type => :string, :aliases => ["-c", "--c"], :default => "/usr/local/cnvrg/run_ipython.sh"
@@ -2904,7 +3160,7 @@ module Cnvrg
2904
3160
  end
2905
3161
  end
2906
3162
 
2907
- desc '', ''
3163
+ desc '', '', :hide => true
2908
3164
  method_option :login, :type => :string, :aliases => ["-l", "--l"], :default => ""
2909
3165
 
2910
3166
  def config_flask_remote(image_name, port=80)
@@ -2959,7 +3215,7 @@ module Cnvrg
2959
3215
  end
2960
3216
  end
2961
3217
 
2962
- desc '', ''
3218
+ desc '', '', :hide => true
2963
3219
 
2964
3220
  def upload_cnvrg_image(image_name)
2965
3221
  verify_logged_in(false)
@@ -3020,7 +3276,7 @@ module Cnvrg
3020
3276
  end
3021
3277
  end
3022
3278
 
3023
- desc '', ''
3279
+ desc '', '', :hide => true
3024
3280
 
3025
3281
  def download_image(image_name, image_slug)
3026
3282
  begin
@@ -3397,7 +3653,6 @@ module Cnvrg
3397
3653
  def self.is_response_success(response, should_exit=true)
3398
3654
  if response.nil?
3399
3655
  if !Cnvrg::Helpers.internet_connection?
3400
- # Cnvrg::CLI.log_end(1,"no internet connection")
3401
3656
  say("<%= color('Error:You seems to be offline', RED) %>")
3402
3657
  else
3403
3658
  say("<%= color('Error', RED) %>")
@@ -3543,11 +3798,15 @@ module Cnvrg
3543
3798
  def is_cnvrg_dir(dir=Dir.pwd)
3544
3799
  current_dir = dir
3545
3800
  home_dir = File.expand_path('~')
3801
+ if current_dir.eql? home_dir
3802
+ return false
3803
+ end
3546
3804
  is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
3547
3805
  until is_cnvrg == true
3548
3806
  current_dir = File.expand_path("..", current_dir)
3549
3807
  is_cnvrg = Dir.exist? current_dir+"/.cnvrg"
3550
- if File.expand_path("..", current_dir).eql? home_dir
3808
+ if ((File.expand_path("..", current_dir).eql? home_dir) or current_dir.eql? home_dir or current_dir.eql? "/") and !is_cnvrg
3809
+ is_cnvrg = false
3551
3810
  break
3552
3811
  end
3553
3812
  end
@@ -3753,6 +4012,16 @@ module Cnvrg
3753
4012
 
3754
4013
  return final
3755
4014
  end
4015
+ def get_cmd_path_in_dir(main_dir,sub_dir)
4016
+ first = Pathname.new main_dir
4017
+ second = Pathname.new sub_dir
4018
+ relative = second.relative_path_from first
4019
+ if relative.eql? "."
4020
+ return ""
4021
+ else
4022
+ return relative
4023
+ end
4024
+ end
3756
4025
 
3757
4026
 
3758
4027