cnvrg 0.0.143 → 0.0.145
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +5 -5
- data/lib/cnvrg/Images.rb +0 -3
- data/lib/cnvrg/api.rb +0 -1
- data/lib/cnvrg/cli.rb +128 -30
- data/lib/cnvrg/experiment.rb +10 -2
- data/lib/cnvrg/files.rb +0 -6
- data/lib/cnvrg/helpers.rb +42 -1
- data/lib/cnvrg/job.rb +0 -1
- data/lib/cnvrg/project.rb +4 -2
- data/lib/cnvrg/version.rb +1 -1
- metadata +24 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b633d9c59001fa185be3d8c960e812a421ecef4e
|
4
|
+
data.tar.gz: bb5b2d41f8d4c4f8a95ce30334cc4e034c2e9bc4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 200acbdc094104d6666b5ad5b1715dd9d241e2c2cd7e3d70d77da0f6e672feb57305d54bd684e721141141201289b3b20aa993bab1bc04d43ea8465dac41eb8c
|
7
|
+
data.tar.gz: 4a79957cb3800306ca5e15cd849ebb02afefd11913eca6f9f7b0145d8133f635c111e6363fdbe0390241405874fc4465c8744afc704dd822517742502ecb8408
|
data/cnvrg.gemspec
CHANGED
@@ -34,11 +34,11 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_runtime_dependency 'aws-sdk'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
|
-
spec.add_runtime_dependency 'logstash-logger'
|
38
|
-
spec.add_runtime_dependency 'launchy'
|
39
|
-
spec.add_runtime_dependency 'docker-api'
|
40
|
-
spec.add_runtime_dependency 'rubyzip'
|
41
|
-
spec.add_runtime_dependency 'activesupport'
|
37
|
+
spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
|
38
|
+
spec.add_runtime_dependency 'launchy', '~> 2.4'
|
39
|
+
spec.add_runtime_dependency 'docker-api', '~> 1.33'
|
40
|
+
spec.add_runtime_dependency 'rubyzip', '~> 1.2'
|
41
|
+
spec.add_runtime_dependency 'activesupport', '~> 5.0'
|
42
42
|
|
43
43
|
|
44
44
|
end
|
data/lib/cnvrg/Images.rb
CHANGED
@@ -46,7 +46,6 @@ module Cnvrg
|
|
46
46
|
end
|
47
47
|
|
48
48
|
rescue => e
|
49
|
-
puts e
|
50
49
|
end
|
51
50
|
|
52
51
|
end
|
@@ -226,7 +225,6 @@ module Cnvrg
|
|
226
225
|
if e.message.include? "is not running"
|
227
226
|
return create_container(port-1)
|
228
227
|
end
|
229
|
-
puts e
|
230
228
|
return false
|
231
229
|
rescue SignalException
|
232
230
|
|
@@ -246,7 +244,6 @@ module Cnvrg
|
|
246
244
|
File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
|
247
245
|
File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
|
248
246
|
rescue => e
|
249
|
-
puts e
|
250
247
|
end
|
251
248
|
|
252
249
|
|
data/lib/cnvrg/api.rb
CHANGED
data/lib/cnvrg/cli.rb
CHANGED
@@ -7,7 +7,7 @@ require 'uri'
|
|
7
7
|
require 'open-uri'
|
8
8
|
require 'json'
|
9
9
|
require 'yaml'
|
10
|
-
require 'digest' #
|
10
|
+
require 'digest' # sha1up
|
11
11
|
require "highline/import"
|
12
12
|
require 'socket'
|
13
13
|
include Open4
|
@@ -61,7 +61,7 @@ module Cnvrg
|
|
61
61
|
desc "", ""
|
62
62
|
method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
|
63
63
|
|
64
|
-
def
|
64
|
+
def testt
|
65
65
|
# image_settings = {
|
66
66
|
# 'Image' => "cnvrg:latest",
|
67
67
|
# 'User' => 'ds',
|
@@ -94,7 +94,7 @@ module Cnvrg
|
|
94
94
|
|
95
95
|
map %w(-v --version) => :version
|
96
96
|
|
97
|
-
desc 'set api url
|
97
|
+
desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
|
98
98
|
|
99
99
|
def set_api_url(url)
|
100
100
|
home_dir = File.expand_path('~')
|
@@ -161,8 +161,8 @@ module Cnvrg
|
|
161
161
|
|
162
162
|
map %w(-api --api) => :set_api_url
|
163
163
|
|
164
|
-
desc '
|
165
|
-
|
164
|
+
desc 'set_default_owner', 'set default owner'
|
165
|
+
|
166
166
|
def set_default_owner
|
167
167
|
|
168
168
|
path = File.expand_path('~')+"/.cnvrg/config.yml"
|
@@ -211,7 +211,7 @@ module Cnvrg
|
|
211
211
|
map %w(--set-default-owner) => :set_default_owner
|
212
212
|
|
213
213
|
|
214
|
-
desc 'login', 'Authenticate with cnvrg.io
|
214
|
+
desc 'login', 'Authenticate with cnvrg.io platform'
|
215
215
|
|
216
216
|
def login
|
217
217
|
begin
|
@@ -467,7 +467,7 @@ module Cnvrg
|
|
467
467
|
@image.update_image_activity(nil, nil)
|
468
468
|
|
469
469
|
end
|
470
|
-
desc 'link', 'Link current directory to a cnvrg project'
|
470
|
+
desc 'link', 'Link current directory to a new cnvrg project'
|
471
471
|
method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
|
472
472
|
method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
|
473
473
|
|
@@ -550,7 +550,7 @@ module Cnvrg
|
|
550
550
|
exit(1)
|
551
551
|
end
|
552
552
|
end
|
553
|
-
desc 'unlink','Unlink a
|
553
|
+
desc 'unlink','Unlink a project from current directory'
|
554
554
|
def unlink
|
555
555
|
verify_logged_in(false)
|
556
556
|
log_start(__method__, args, options)
|
@@ -559,7 +559,7 @@ module Cnvrg
|
|
559
559
|
FileUtils.rm_rf list_to_del
|
560
560
|
end
|
561
561
|
|
562
|
-
desc 'clone', 'Clone
|
562
|
+
desc 'clone', 'Clone project'
|
563
563
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
|
564
564
|
method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
|
565
565
|
|
@@ -687,6 +687,7 @@ module Cnvrg
|
|
687
687
|
commit = result["commit"]
|
688
688
|
result = result["tree"]
|
689
689
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
690
|
+
|
690
691
|
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
691
692
|
say "Project is up to date", Thor::Shell::Color::GREEN
|
692
693
|
log_end(0)
|
@@ -780,6 +781,7 @@ module Cnvrg
|
|
780
781
|
end
|
781
782
|
end
|
782
783
|
result = @project.compare_idx(new_branch)
|
784
|
+
|
783
785
|
commit = result["result"]["commit"]
|
784
786
|
if !link
|
785
787
|
if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
@@ -792,6 +794,19 @@ module Cnvrg
|
|
792
794
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
793
795
|
end
|
794
796
|
result = result["result"]["tree"]
|
797
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
798
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
799
|
+
# if all.size == 1
|
800
|
+
# num = "conflict"
|
801
|
+
# else
|
802
|
+
# num = "conflicts"
|
803
|
+
# end
|
804
|
+
# say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
|
805
|
+
# say "#{all.join("\n")}"
|
806
|
+
# say "Please fix #{num}, and retry", Thor::Shell::Color::RED
|
807
|
+
# exit(1)
|
808
|
+
#
|
809
|
+
# end
|
795
810
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
796
811
|
log_end(0)
|
797
812
|
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
@@ -842,21 +857,21 @@ module Cnvrg
|
|
842
857
|
end
|
843
858
|
|
844
859
|
# delete
|
845
|
-
result["deleted"]
|
860
|
+
deleted = update_deleted(result["deleted"])
|
861
|
+
deleted.each do |f|
|
846
862
|
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
847
863
|
if relative_path.end_with?("/")
|
848
864
|
if @files.delete_dir(f, relative_path, commit_sha1)
|
849
|
-
update_count += 1
|
865
|
+
# update_count += 1
|
850
866
|
successful_updates<< relative_path
|
851
867
|
end
|
852
868
|
else
|
853
869
|
if @files.delete_file(f, relative_path, commit_sha1)
|
854
|
-
update_count += 1
|
870
|
+
# update_count += 1
|
855
871
|
successful_updates<< relative_path
|
856
872
|
end
|
857
873
|
end
|
858
874
|
end
|
859
|
-
log_end(0)
|
860
875
|
|
861
876
|
rescue SignalException
|
862
877
|
log_end(-1)
|
@@ -869,6 +884,9 @@ module Cnvrg
|
|
869
884
|
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
870
885
|
exit(0)
|
871
886
|
end
|
887
|
+
if !result["deleted"].nil? and !result["deleted"].empty?
|
888
|
+
update_count += result["deleted"].size
|
889
|
+
end
|
872
890
|
if update_count == update_total
|
873
891
|
res = @files.end_commit(commit_sha1)
|
874
892
|
if (Cnvrg::CLI.is_response_success(res, false))
|
@@ -904,7 +922,13 @@ module Cnvrg
|
|
904
922
|
end
|
905
923
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
906
924
|
else
|
907
|
-
|
925
|
+
if !options["sync"] and !sync
|
926
|
+
# say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
927
|
+
|
928
|
+
# else
|
929
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
930
|
+
|
931
|
+
end
|
908
932
|
|
909
933
|
end
|
910
934
|
|
@@ -916,10 +940,12 @@ module Cnvrg
|
|
916
940
|
end
|
917
941
|
else
|
918
942
|
log_end(1, "error. Rolling Back all changes")
|
943
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::BLUE
|
919
944
|
|
920
945
|
@files.rollback_commit(commit_sha1)
|
921
946
|
end
|
922
|
-
rescue
|
947
|
+
rescue =>e
|
948
|
+
|
923
949
|
log_end(-1)
|
924
950
|
|
925
951
|
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
@@ -941,7 +967,7 @@ module Cnvrg
|
|
941
967
|
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
942
968
|
method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
|
943
969
|
|
944
|
-
def download
|
970
|
+
def download(sync=false)
|
945
971
|
begin
|
946
972
|
verify_logged_in(true)
|
947
973
|
log_start(__method__, args, options)
|
@@ -951,13 +977,27 @@ module Cnvrg
|
|
951
977
|
new_branch = options["new_branch"] || false
|
952
978
|
|
953
979
|
res = @project.compare_idx(new_branch)["result"]
|
980
|
+
|
954
981
|
result = res["tree"]
|
982
|
+
|
955
983
|
commit = res["commit"]
|
956
|
-
if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
|
957
|
-
say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
|
984
|
+
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
985
|
+
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
958
986
|
log_end(0)
|
959
987
|
return true
|
960
988
|
end
|
989
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
990
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
991
|
+
# if all.size == 1
|
992
|
+
# num = "1 conflict"
|
993
|
+
# else
|
994
|
+
# num = "#{result["conflicts"].size} conflicts"
|
995
|
+
# end
|
996
|
+
# say "Project contains #{num}:", Thor::Shell::Color::RED
|
997
|
+
# say "#{all.join("\n")}"
|
998
|
+
# say "Please fix them, and retry", Thor::Shell::Color::RED
|
999
|
+
# exit(1)
|
1000
|
+
# end
|
961
1001
|
update_count = 0
|
962
1002
|
update_total = result["updated_on_server"].size + result["conflicts"].size
|
963
1003
|
|
@@ -1010,12 +1050,35 @@ module Cnvrg
|
|
1010
1050
|
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
1011
1051
|
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
1012
1052
|
else
|
1013
|
-
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
1053
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN if !sync
|
1014
1054
|
end
|
1015
1055
|
|
1016
1056
|
|
1017
1057
|
log_end(0)
|
1018
1058
|
end
|
1059
|
+
rescue
|
1060
|
+
log_end(-1)
|
1061
|
+
|
1062
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
1063
|
+
if successful_changes.nil?
|
1064
|
+
exit(1)
|
1065
|
+
end
|
1066
|
+
successful_changes.each do |f|
|
1067
|
+
|
1068
|
+
abs_path = "#{@project.local_path}/#{f}"
|
1069
|
+
filename = File.basename abs_path
|
1070
|
+
say "revoking #{filename}"
|
1071
|
+
if result["conflicts"].include? f
|
1072
|
+
@files.revoke_download_file(abs_path, f, filename, true)
|
1073
|
+
elsif result["updated_on_server"].include? f
|
1074
|
+
if File.directory? abs_path
|
1075
|
+
@files.revoke_download_dir(abs_path, f, project_home)
|
1076
|
+
else
|
1077
|
+
@files.revoke_download_file(project_home, abs_path, filename)
|
1078
|
+
end
|
1079
|
+
end
|
1080
|
+
end
|
1081
|
+
exit(1)
|
1019
1082
|
rescue SignalException
|
1020
1083
|
log_end(-1)
|
1021
1084
|
say "\nAborting", Thor::Shell::Color::BLUE
|
@@ -1113,9 +1176,7 @@ module Cnvrg
|
|
1113
1176
|
|
1114
1177
|
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
|
1115
1178
|
invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
|
1116
|
-
check = Helpers.checkmark()
|
1117
1179
|
|
1118
|
-
say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
1119
1180
|
|
1120
1181
|
end
|
1121
1182
|
|
@@ -1127,6 +1188,7 @@ module Cnvrg
|
|
1127
1188
|
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
1128
1189
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1129
1190
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
1191
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
1130
1192
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
|
1131
1193
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
|
1132
1194
|
method_option :title, :type => :string, :aliases => ["-t"], :default => ""
|
@@ -1136,6 +1198,7 @@ module Cnvrg
|
|
1136
1198
|
method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
|
1137
1199
|
method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
|
1138
1200
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1201
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
1139
1202
|
|
1140
1203
|
def run(*cmd)
|
1141
1204
|
sync_before = options["sync_before"]
|
@@ -1148,15 +1211,17 @@ module Cnvrg
|
|
1148
1211
|
local = options["local"]
|
1149
1212
|
schedule = options["schedule"]
|
1150
1213
|
image = options["image"]
|
1214
|
+
grid = options["grid"]
|
1151
1215
|
if local
|
1152
1216
|
invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
|
1153
1217
|
:log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
|
1154
1218
|
return
|
1155
1219
|
else
|
1156
|
-
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1220
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1221
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
1157
1222
|
instance_type = get_instance_type(instances)
|
1158
1223
|
invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
1159
|
-
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image
|
1224
|
+
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image, :grid => grid
|
1160
1225
|
return
|
1161
1226
|
end
|
1162
1227
|
end
|
@@ -1315,6 +1380,8 @@ module Cnvrg
|
|
1315
1380
|
say "The process exited!", Thor::Shell::Color::RED
|
1316
1381
|
rescue => e
|
1317
1382
|
log_end(-1, e.message)
|
1383
|
+
res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
|
1384
|
+
|
1318
1385
|
say "Error occurred,aborting", Thor::Shell::Color::RED
|
1319
1386
|
exit(0)
|
1320
1387
|
end
|
@@ -1327,7 +1394,7 @@ module Cnvrg
|
|
1327
1394
|
end
|
1328
1395
|
if !exp_success
|
1329
1396
|
if !Cnvrg::Helpers.internet_connection?
|
1330
|
-
wait_offline = agree "Seems like you're offline, wait until
|
1397
|
+
wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
|
1331
1398
|
if wait_offline
|
1332
1399
|
say "Waiting until your'e online..", Thor::Shell::Color::BLUE
|
1333
1400
|
while !Cnvrg::Helpers.internet_connection?
|
@@ -1351,7 +1418,7 @@ module Cnvrg
|
|
1351
1418
|
if sync_after
|
1352
1419
|
# Sync after run
|
1353
1420
|
|
1354
|
-
download()
|
1421
|
+
download(sync=true)
|
1355
1422
|
upload(link=false, sync=true)
|
1356
1423
|
|
1357
1424
|
end
|
@@ -1363,7 +1430,6 @@ module Cnvrg
|
|
1363
1430
|
log_end(0)
|
1364
1431
|
end
|
1365
1432
|
rescue => e
|
1366
|
-
puts e
|
1367
1433
|
log_end(-1, e.message)
|
1368
1434
|
if container
|
1369
1435
|
container.stop()
|
@@ -1587,12 +1653,14 @@ module Cnvrg
|
|
1587
1653
|
method_option :schedule, :type => :string, :aliases => [ "-s"], :default => ""
|
1588
1654
|
method_option :commit, :type => :string, :aliases => [ "-c"], :default => nil
|
1589
1655
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1656
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
1590
1657
|
|
1591
1658
|
def exec_remote(*cmd)
|
1592
1659
|
verify_logged_in(true)
|
1593
1660
|
log_start(__method__, args, options)
|
1594
1661
|
working_dir = is_cnvrg_dir
|
1595
1662
|
begin
|
1663
|
+
grid = options["grid"] || nil
|
1596
1664
|
|
1597
1665
|
instance_type = options["machine_type"] || nil
|
1598
1666
|
schedule = options["schedule"] || ""
|
@@ -1622,7 +1690,7 @@ module Cnvrg
|
|
1622
1690
|
upload_output_option = "--upload_output=#{upload_output}"
|
1623
1691
|
end
|
1624
1692
|
options_hash = Hash[options]
|
1625
|
-
options_hash.except!("schedule", "machine_type", "image", "upload_output")
|
1693
|
+
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid")
|
1626
1694
|
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
1627
1695
|
command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
1628
1696
|
commit_to_run = options["commit"] || nil
|
@@ -1663,7 +1731,7 @@ module Cnvrg
|
|
1663
1731
|
say "Running remote experiment", Thor::Shell::Color::BLUE
|
1664
1732
|
exp = Experiment.new(project.owner, project.slug)
|
1665
1733
|
|
1666
|
-
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
1734
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid)
|
1667
1735
|
if Cnvrg::CLI.is_response_success(res)
|
1668
1736
|
|
1669
1737
|
# if res["result"]["machine"] == -1
|
@@ -1697,7 +1765,13 @@ module Cnvrg
|
|
1697
1765
|
# end
|
1698
1766
|
# else
|
1699
1767
|
check = Helpers.checkmark()
|
1700
|
-
|
1768
|
+
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
1769
|
+
|
1770
|
+
if res["result"]["grid"]
|
1771
|
+
str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
|
1772
|
+
end
|
1773
|
+
|
1774
|
+
say str, Thor::Shell::Color::GREEN
|
1701
1775
|
|
1702
1776
|
exit(0)
|
1703
1777
|
# end
|
@@ -1821,6 +1895,7 @@ module Cnvrg
|
|
1821
1895
|
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
1822
1896
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1823
1897
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
1898
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
1824
1899
|
method_option :image, :type => :string, :aliases => ["--i"], :default => ""
|
1825
1900
|
|
1826
1901
|
desc 'starts a notebook session', 'starts a notebook session remotely or locally'
|
@@ -1835,7 +1910,8 @@ module Cnvrg
|
|
1835
1910
|
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel,:image =>image
|
1836
1911
|
return
|
1837
1912
|
else
|
1838
|
-
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1913
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1914
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
1839
1915
|
instance_type = get_instance_type(instances)
|
1840
1916
|
|
1841
1917
|
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image =>image
|
@@ -3656,6 +3732,28 @@ module Cnvrg
|
|
3656
3732
|
return new_time
|
3657
3733
|
|
3658
3734
|
end
|
3735
|
+
def update_deleted(deleted)
|
3736
|
+
final = []
|
3737
|
+
deleted.each do |d|
|
3738
|
+
all_subs = d.split("/")
|
3739
|
+
to_add = true
|
3740
|
+
value = all_subs[0]
|
3741
|
+
all_subs.each_with_index do |a,i|
|
3742
|
+
if final.include? value+"/"
|
3743
|
+
to_add = false
|
3744
|
+
break
|
3745
|
+
end
|
3746
|
+
value = value+"/"+all_subs[i+1] if i <all_subs.size-1
|
3747
|
+
end
|
3748
|
+
final << d if to_add
|
3749
|
+
|
3750
|
+
|
3751
|
+
end
|
3752
|
+
|
3753
|
+
|
3754
|
+
return final
|
3755
|
+
end
|
3756
|
+
|
3659
3757
|
|
3660
3758
|
|
3661
3759
|
end
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -78,12 +78,13 @@ module Cnvrg
|
|
78
78
|
|
79
79
|
end
|
80
80
|
|
81
|
-
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
|
81
|
+
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid)
|
82
82
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
83
83
|
commit_sha1: commit_to_run,
|
84
84
|
instance_type: instance_type,
|
85
85
|
scheduling_query:scheduling_query,
|
86
|
-
local_timestamp:local_timestamp
|
86
|
+
local_timestamp:local_timestamp,
|
87
|
+
grid: grid})
|
87
88
|
return response
|
88
89
|
end
|
89
90
|
|
@@ -95,10 +96,17 @@ module Cnvrg
|
|
95
96
|
end
|
96
97
|
|
97
98
|
def end(output, exit_status, end_commit, cpu_average, memory_average)
|
99
|
+
#if remote try to remove
|
98
100
|
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
|
99
101
|
exit_status: exit_status, end_commit: end_commit,
|
100
102
|
cpu_average: cpu_average, memory_average: memory_average})
|
101
103
|
Cnvrg::CLI.is_response_success(response)
|
104
|
+
|
105
|
+
begin
|
106
|
+
FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
|
107
|
+
rescue
|
108
|
+
|
109
|
+
end
|
102
110
|
end
|
103
111
|
end
|
104
112
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -130,7 +130,6 @@ module Cnvrg
|
|
130
130
|
end
|
131
131
|
return false
|
132
132
|
rescue =>e
|
133
|
-
puts e
|
134
133
|
end
|
135
134
|
|
136
135
|
end
|
@@ -171,7 +170,6 @@ module Cnvrg
|
|
171
170
|
end
|
172
171
|
return false
|
173
172
|
rescue =>e
|
174
|
-
puts e
|
175
173
|
end
|
176
174
|
|
177
175
|
end
|
@@ -190,7 +188,6 @@ module Cnvrg
|
|
190
188
|
|
191
189
|
return true
|
192
190
|
rescue => e
|
193
|
-
puts e
|
194
191
|
return false
|
195
192
|
end
|
196
193
|
|
@@ -226,7 +223,6 @@ module Cnvrg
|
|
226
223
|
upload_file(file_path)
|
227
224
|
return resp
|
228
225
|
rescue =>e
|
229
|
-
puts e
|
230
226
|
return false
|
231
227
|
|
232
228
|
end
|
@@ -310,8 +306,6 @@ module Cnvrg
|
|
310
306
|
end
|
311
307
|
|
312
308
|
rescue =>e
|
313
|
-
puts e
|
314
|
-
puts e.backtrace
|
315
309
|
return false
|
316
310
|
|
317
311
|
end
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -72,7 +72,7 @@ module Cnvrg
|
|
72
72
|
|
73
73
|
def cnvrgignore_content
|
74
74
|
%{
|
75
|
-
|
75
|
+
# cnvrg ignore: Ignore the following directories and files
|
76
76
|
# for example:
|
77
77
|
# some_dir/
|
78
78
|
# some_file.txt
|
@@ -81,6 +81,47 @@ module Cnvrg
|
|
81
81
|
}.strip
|
82
82
|
end
|
83
83
|
|
84
|
+
def hyper_content
|
85
|
+
%{# Hyperparameter Optimization is the process of choosing a set of parameters for a learning algorithm, usually with the goal of optimizing a measure of the algorithm's performance on an independent data set.
|
86
|
+
|
87
|
+
# Below is the list of parameters that will be used in the optimization process. Each parameter has a param_name that should match the argument that is feeded to the experiment s.t kernel => --kernel='rbf'
|
88
|
+
|
89
|
+
parameters:
|
90
|
+
# Integer parameter is a range of possible values between a minimum (inclusive)
|
91
|
+
# and maximum (not inclusive) values. Values are floored (0.7 => 0)
|
92
|
+
- param_name: "learning_rate"
|
93
|
+
type: "integer"
|
94
|
+
min: 0 # inclusive
|
95
|
+
max: 10 # not inclusive
|
96
|
+
scale: "linear"
|
97
|
+
steps: 4 # The number of linear steps to produce.
|
98
|
+
|
99
|
+
|
100
|
+
# Float parameter is a range of possible values between a minimum (inclusive)
|
101
|
+
# and maximum (not inclusive) values.
|
102
|
+
#
|
103
|
+
- param_name: "learning_rate"
|
104
|
+
type: "float" # precision is 9 after period
|
105
|
+
min: 0.00001
|
106
|
+
max: 0.1
|
107
|
+
scale: "log2" # Could be log10 as well
|
108
|
+
steps: 2
|
109
|
+
|
110
|
+
# Discrete parameter is an array of numerical values.
|
111
|
+
#
|
112
|
+
- param_name: "c"
|
113
|
+
type: "discrete"
|
114
|
+
values: [0, 0.1 ,0.001]
|
115
|
+
|
116
|
+
# Categorical parameter is an array of string values
|
117
|
+
#
|
118
|
+
- param_name: "kernel"
|
119
|
+
type: "categorical"
|
120
|
+
values: ["linear", "poly", "rbf"]
|
121
|
+
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
84
125
|
def readme_content
|
85
126
|
%{
|
86
127
|
# README
|
data/lib/cnvrg/job.rb
CHANGED
data/lib/cnvrg/project.rb
CHANGED
@@ -90,6 +90,7 @@ module Cnvrg
|
|
90
90
|
]
|
91
91
|
cnvrgreadme = Helpers.readme_content
|
92
92
|
cnvrgignore = Helpers.cnvrgignore_content
|
93
|
+
cnvrghyper = Helpers.hyper_content
|
93
94
|
|
94
95
|
begin
|
95
96
|
|
@@ -109,6 +110,8 @@ module Cnvrg
|
|
109
110
|
File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
110
111
|
File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
111
112
|
File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
|
113
|
+
File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
|
114
|
+
|
112
115
|
rescue
|
113
116
|
return false
|
114
117
|
end
|
@@ -137,14 +140,13 @@ module Cnvrg
|
|
137
140
|
FileUtils.mkdir_p list_dirs
|
138
141
|
FileUtils.touch list_files
|
139
142
|
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
140
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
143
|
+
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
|
141
144
|
if !File.exist? "README" and !File.exist? "README.md"
|
142
145
|
FileUtils.touch [ "README.md" ]
|
143
146
|
File.open("README.md", "w+") { |f| f.write cnvrgreadme }
|
144
147
|
end
|
145
148
|
|
146
149
|
rescue => e
|
147
|
-
puts e
|
148
150
|
return false
|
149
151
|
end
|
150
152
|
return true
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.145
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-04-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -249,72 +249,72 @@ dependencies:
|
|
249
249
|
name: logstash-logger
|
250
250
|
requirement: !ruby/object:Gem::Requirement
|
251
251
|
requirements:
|
252
|
-
- - "
|
252
|
+
- - "~>"
|
253
253
|
- !ruby/object:Gem::Version
|
254
|
-
version:
|
254
|
+
version: 0.22.1
|
255
255
|
type: :runtime
|
256
256
|
prerelease: false
|
257
257
|
version_requirements: !ruby/object:Gem::Requirement
|
258
258
|
requirements:
|
259
|
-
- - "
|
259
|
+
- - "~>"
|
260
260
|
- !ruby/object:Gem::Version
|
261
|
-
version:
|
261
|
+
version: 0.22.1
|
262
262
|
- !ruby/object:Gem::Dependency
|
263
263
|
name: launchy
|
264
264
|
requirement: !ruby/object:Gem::Requirement
|
265
265
|
requirements:
|
266
|
-
- - "
|
266
|
+
- - "~>"
|
267
267
|
- !ruby/object:Gem::Version
|
268
|
-
version: '
|
268
|
+
version: '2.4'
|
269
269
|
type: :runtime
|
270
270
|
prerelease: false
|
271
271
|
version_requirements: !ruby/object:Gem::Requirement
|
272
272
|
requirements:
|
273
|
-
- - "
|
273
|
+
- - "~>"
|
274
274
|
- !ruby/object:Gem::Version
|
275
|
-
version: '
|
275
|
+
version: '2.4'
|
276
276
|
- !ruby/object:Gem::Dependency
|
277
277
|
name: docker-api
|
278
278
|
requirement: !ruby/object:Gem::Requirement
|
279
279
|
requirements:
|
280
|
-
- - "
|
280
|
+
- - "~>"
|
281
281
|
- !ruby/object:Gem::Version
|
282
|
-
version: '
|
282
|
+
version: '1.33'
|
283
283
|
type: :runtime
|
284
284
|
prerelease: false
|
285
285
|
version_requirements: !ruby/object:Gem::Requirement
|
286
286
|
requirements:
|
287
|
-
- - "
|
287
|
+
- - "~>"
|
288
288
|
- !ruby/object:Gem::Version
|
289
|
-
version: '
|
289
|
+
version: '1.33'
|
290
290
|
- !ruby/object:Gem::Dependency
|
291
291
|
name: rubyzip
|
292
292
|
requirement: !ruby/object:Gem::Requirement
|
293
293
|
requirements:
|
294
|
-
- - "
|
294
|
+
- - "~>"
|
295
295
|
- !ruby/object:Gem::Version
|
296
|
-
version: '
|
296
|
+
version: '1.2'
|
297
297
|
type: :runtime
|
298
298
|
prerelease: false
|
299
299
|
version_requirements: !ruby/object:Gem::Requirement
|
300
300
|
requirements:
|
301
|
-
- - "
|
301
|
+
- - "~>"
|
302
302
|
- !ruby/object:Gem::Version
|
303
|
-
version: '
|
303
|
+
version: '1.2'
|
304
304
|
- !ruby/object:Gem::Dependency
|
305
305
|
name: activesupport
|
306
306
|
requirement: !ruby/object:Gem::Requirement
|
307
307
|
requirements:
|
308
|
-
- - "
|
308
|
+
- - "~>"
|
309
309
|
- !ruby/object:Gem::Version
|
310
|
-
version: '0'
|
310
|
+
version: '5.0'
|
311
311
|
type: :runtime
|
312
312
|
prerelease: false
|
313
313
|
version_requirements: !ruby/object:Gem::Requirement
|
314
314
|
requirements:
|
315
|
-
- - "
|
315
|
+
- - "~>"
|
316
316
|
- !ruby/object:Gem::Version
|
317
|
-
version: '0'
|
317
|
+
version: '5.0'
|
318
318
|
description: A CLI tool for interacting with cnvrg.io.
|
319
319
|
email:
|
320
320
|
- info@cnvrg.io
|
@@ -356,8 +356,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
356
356
|
version: '0'
|
357
357
|
requirements: []
|
358
358
|
rubyforge_project:
|
359
|
-
rubygems_version: 2.
|
359
|
+
rubygems_version: 2.5.1
|
360
360
|
signing_key:
|
361
361
|
specification_version: 4
|
362
362
|
summary: A CLI tool for interacting with cnvrg.io.
|
363
363
|
test_files: []
|
364
|
+
has_rdoc:
|