cnvrg 0.0.143 → 0.0.145
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cnvrg.gemspec +5 -5
- data/lib/cnvrg/Images.rb +0 -3
- data/lib/cnvrg/api.rb +0 -1
- data/lib/cnvrg/cli.rb +128 -30
- data/lib/cnvrg/experiment.rb +10 -2
- data/lib/cnvrg/files.rb +0 -6
- data/lib/cnvrg/helpers.rb +42 -1
- data/lib/cnvrg/job.rb +0 -1
- data/lib/cnvrg/project.rb +4 -2
- data/lib/cnvrg/version.rb +1 -1
- metadata +24 -23
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b633d9c59001fa185be3d8c960e812a421ecef4e
|
4
|
+
data.tar.gz: bb5b2d41f8d4c4f8a95ce30334cc4e034c2e9bc4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 200acbdc094104d6666b5ad5b1715dd9d241e2c2cd7e3d70d77da0f6e672feb57305d54bd684e721141141201289b3b20aa993bab1bc04d43ea8465dac41eb8c
|
7
|
+
data.tar.gz: 4a79957cb3800306ca5e15cd849ebb02afefd11913eca6f9f7b0145d8133f635c111e6363fdbe0390241405874fc4465c8744afc704dd822517742502ecb8408
|
data/cnvrg.gemspec
CHANGED
@@ -34,11 +34,11 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_runtime_dependency 'aws-sdk'
|
35
35
|
spec.add_runtime_dependency 'sucker_punch', '~> 2.0'
|
36
36
|
spec.add_runtime_dependency 'urlcrypt', '~> 0.1.1'
|
37
|
-
spec.add_runtime_dependency 'logstash-logger'
|
38
|
-
spec.add_runtime_dependency 'launchy'
|
39
|
-
spec.add_runtime_dependency 'docker-api'
|
40
|
-
spec.add_runtime_dependency 'rubyzip'
|
41
|
-
spec.add_runtime_dependency 'activesupport'
|
37
|
+
spec.add_runtime_dependency 'logstash-logger', '~> 0.22.1'
|
38
|
+
spec.add_runtime_dependency 'launchy', '~> 2.4'
|
39
|
+
spec.add_runtime_dependency 'docker-api', '~> 1.33'
|
40
|
+
spec.add_runtime_dependency 'rubyzip', '~> 1.2'
|
41
|
+
spec.add_runtime_dependency 'activesupport', '~> 5.0'
|
42
42
|
|
43
43
|
|
44
44
|
end
|
data/lib/cnvrg/Images.rb
CHANGED
@@ -46,7 +46,6 @@ module Cnvrg
|
|
46
46
|
end
|
47
47
|
|
48
48
|
rescue => e
|
49
|
-
puts e
|
50
49
|
end
|
51
50
|
|
52
51
|
end
|
@@ -226,7 +225,6 @@ module Cnvrg
|
|
226
225
|
if e.message.include? "is not running"
|
227
226
|
return create_container(port-1)
|
228
227
|
end
|
229
|
-
puts e
|
230
228
|
return false
|
231
229
|
rescue SignalException
|
232
230
|
|
@@ -246,7 +244,6 @@ module Cnvrg
|
|
246
244
|
File.open(@working_dir+"/.cnvrg/pip_base.txt", "w+") { |f| f.write pip }
|
247
245
|
File.open(@working_dir+"/.cnvrg/dpkg_base.txt", "w+") { |f| f.write dpkg }
|
248
246
|
rescue => e
|
249
|
-
puts e
|
250
247
|
end
|
251
248
|
|
252
249
|
|
data/lib/cnvrg/api.rb
CHANGED
data/lib/cnvrg/cli.rb
CHANGED
@@ -7,7 +7,7 @@ require 'uri'
|
|
7
7
|
require 'open-uri'
|
8
8
|
require 'json'
|
9
9
|
require 'yaml'
|
10
|
-
require 'digest' #
|
10
|
+
require 'digest' # sha1up
|
11
11
|
require "highline/import"
|
12
12
|
require 'socket'
|
13
13
|
include Open4
|
@@ -61,7 +61,7 @@ module Cnvrg
|
|
61
61
|
desc "", ""
|
62
62
|
method_option :schedule, :type => :string, :aliases => ["--s", "-s"], :default => "leahs"
|
63
63
|
|
64
|
-
def
|
64
|
+
def testt
|
65
65
|
# image_settings = {
|
66
66
|
# 'Image' => "cnvrg:latest",
|
67
67
|
# 'User' => 'ds',
|
@@ -94,7 +94,7 @@ module Cnvrg
|
|
94
94
|
|
95
95
|
map %w(-v --version) => :version
|
96
96
|
|
97
|
-
desc 'set api url
|
97
|
+
desc 'api', 'set api url, e.g cnvrg --api "https://cnvrg.io/api"'
|
98
98
|
|
99
99
|
def set_api_url(url)
|
100
100
|
home_dir = File.expand_path('~')
|
@@ -161,8 +161,8 @@ module Cnvrg
|
|
161
161
|
|
162
162
|
map %w(-api --api) => :set_api_url
|
163
163
|
|
164
|
-
desc '
|
165
|
-
|
164
|
+
desc 'set_default_owner', 'set default owner'
|
165
|
+
|
166
166
|
def set_default_owner
|
167
167
|
|
168
168
|
path = File.expand_path('~')+"/.cnvrg/config.yml"
|
@@ -211,7 +211,7 @@ module Cnvrg
|
|
211
211
|
map %w(--set-default-owner) => :set_default_owner
|
212
212
|
|
213
213
|
|
214
|
-
desc 'login', 'Authenticate with cnvrg.io
|
214
|
+
desc 'login', 'Authenticate with cnvrg.io platform'
|
215
215
|
|
216
216
|
def login
|
217
217
|
begin
|
@@ -467,7 +467,7 @@ module Cnvrg
|
|
467
467
|
@image.update_image_activity(nil, nil)
|
468
468
|
|
469
469
|
end
|
470
|
-
desc 'link', 'Link current directory to a cnvrg project'
|
470
|
+
desc 'link', 'Link current directory to a new cnvrg project'
|
471
471
|
method_option :sync, :type => :boolean, :aliases => ["-s", "--s"], :default => true
|
472
472
|
method_option :docker_image, :type => :string, :aliases => ["-d", "--d"], :default => ""
|
473
473
|
|
@@ -550,7 +550,7 @@ module Cnvrg
|
|
550
550
|
exit(1)
|
551
551
|
end
|
552
552
|
end
|
553
|
-
desc 'unlink','Unlink a
|
553
|
+
desc 'unlink','Unlink a project from current directory'
|
554
554
|
def unlink
|
555
555
|
verify_logged_in(false)
|
556
556
|
log_start(__method__, args, options)
|
@@ -559,7 +559,7 @@ module Cnvrg
|
|
559
559
|
FileUtils.rm_rf list_to_del
|
560
560
|
end
|
561
561
|
|
562
|
-
desc 'clone', 'Clone
|
562
|
+
desc 'clone', 'Clone project'
|
563
563
|
method_option :remote, :type => :boolean, :aliases => ["-r", "--r"], :default => false
|
564
564
|
method_option :commit, :type => :string, :aliases => ["-c", "--c"], :default => nil
|
565
565
|
|
@@ -687,6 +687,7 @@ module Cnvrg
|
|
687
687
|
commit = result["commit"]
|
688
688
|
result = result["tree"]
|
689
689
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE
|
690
|
+
|
690
691
|
if result["added"].empty? and result["updated_on_local"].empty? and result["updated_on_server"].empty? and result["deleted"].empty? and result["conflicts"].empty?
|
691
692
|
say "Project is up to date", Thor::Shell::Color::GREEN
|
692
693
|
log_end(0)
|
@@ -780,6 +781,7 @@ module Cnvrg
|
|
780
781
|
end
|
781
782
|
end
|
782
783
|
result = @project.compare_idx(new_branch)
|
784
|
+
|
783
785
|
commit = result["result"]["commit"]
|
784
786
|
if !link
|
785
787
|
if commit != @project.last_local_commit and !@project.last_local_commit.nil? and !result["result"]["tree"]["updated_on_server"].empty?
|
@@ -792,6 +794,19 @@ module Cnvrg
|
|
792
794
|
say "Comparing local changes with remote version:", Thor::Shell::Color::BLUE if options["verbose"]
|
793
795
|
end
|
794
796
|
result = result["result"]["tree"]
|
797
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
798
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
799
|
+
# if all.size == 1
|
800
|
+
# num = "conflict"
|
801
|
+
# else
|
802
|
+
# num = "conflicts"
|
803
|
+
# end
|
804
|
+
# say "Project contains #{all.size} #{num}:", Thor::Shell::Color::RED
|
805
|
+
# say "#{all.join("\n")}"
|
806
|
+
# say "Please fix #{num}, and retry", Thor::Shell::Color::RED
|
807
|
+
# exit(1)
|
808
|
+
#
|
809
|
+
# end
|
795
810
|
if result["added"].empty? and result["updated_on_local"].empty? and result["deleted"].empty?
|
796
811
|
log_end(0)
|
797
812
|
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
@@ -842,21 +857,21 @@ module Cnvrg
|
|
842
857
|
end
|
843
858
|
|
844
859
|
# delete
|
845
|
-
result["deleted"]
|
860
|
+
deleted = update_deleted(result["deleted"])
|
861
|
+
deleted.each do |f|
|
846
862
|
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
847
863
|
if relative_path.end_with?("/")
|
848
864
|
if @files.delete_dir(f, relative_path, commit_sha1)
|
849
|
-
update_count += 1
|
865
|
+
# update_count += 1
|
850
866
|
successful_updates<< relative_path
|
851
867
|
end
|
852
868
|
else
|
853
869
|
if @files.delete_file(f, relative_path, commit_sha1)
|
854
|
-
update_count += 1
|
870
|
+
# update_count += 1
|
855
871
|
successful_updates<< relative_path
|
856
872
|
end
|
857
873
|
end
|
858
874
|
end
|
859
|
-
log_end(0)
|
860
875
|
|
861
876
|
rescue SignalException
|
862
877
|
log_end(-1)
|
@@ -869,6 +884,9 @@ module Cnvrg
|
|
869
884
|
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
870
885
|
exit(0)
|
871
886
|
end
|
887
|
+
if !result["deleted"].nil? and !result["deleted"].empty?
|
888
|
+
update_count += result["deleted"].size
|
889
|
+
end
|
872
890
|
if update_count == update_total
|
873
891
|
res = @files.end_commit(commit_sha1)
|
874
892
|
if (Cnvrg::CLI.is_response_success(res, false))
|
@@ -904,7 +922,13 @@ module Cnvrg
|
|
904
922
|
end
|
905
923
|
say "Total of #{update_count} / #{update_total} files.", Thor::Shell::Color::GREEN
|
906
924
|
else
|
907
|
-
|
925
|
+
if !options["sync"] and !sync
|
926
|
+
# say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
927
|
+
|
928
|
+
# else
|
929
|
+
say "#{check} Changes were updated successfully", Thor::Shell::Color::GREEN
|
930
|
+
|
931
|
+
end
|
908
932
|
|
909
933
|
end
|
910
934
|
|
@@ -916,10 +940,12 @@ module Cnvrg
|
|
916
940
|
end
|
917
941
|
else
|
918
942
|
log_end(1, "error. Rolling Back all changes")
|
943
|
+
say "Error occurd, \nRolling back", Thor::Shell::Color::BLUE
|
919
944
|
|
920
945
|
@files.rollback_commit(commit_sha1)
|
921
946
|
end
|
922
|
-
rescue
|
947
|
+
rescue =>e
|
948
|
+
|
923
949
|
log_end(-1)
|
924
950
|
|
925
951
|
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
@@ -941,7 +967,7 @@ module Cnvrg
|
|
941
967
|
method_option :verbose, :type => :boolean, :aliases => ["--v"], :default => false
|
942
968
|
method_option :sync, :type => :boolean, :aliases => ["--s"], :default => false
|
943
969
|
|
944
|
-
def download
|
970
|
+
def download(sync=false)
|
945
971
|
begin
|
946
972
|
verify_logged_in(true)
|
947
973
|
log_start(__method__, args, options)
|
@@ -951,13 +977,27 @@ module Cnvrg
|
|
951
977
|
new_branch = options["new_branch"] || false
|
952
978
|
|
953
979
|
res = @project.compare_idx(new_branch)["result"]
|
980
|
+
|
954
981
|
result = res["tree"]
|
982
|
+
|
955
983
|
commit = res["commit"]
|
956
|
-
if result["updated_on_server"].empty? and result["conflicts"] and result["deleted"].empty?
|
957
|
-
say "Project is up to date", Thor::Shell::Color::GREEN unless options["sync"]
|
984
|
+
if result["updated_on_server"].empty? and result["conflicts"].empty? and result["deleted"].empty?
|
985
|
+
say "Project is up to date", Thor::Shell::Color::GREEN unless (options["sync"] or sync)
|
958
986
|
log_end(0)
|
959
987
|
return true
|
960
988
|
end
|
989
|
+
# if result["added"].any? {|x| x.include? ".conflict"} or !result["conflicts"].empty?
|
990
|
+
# all = result["added"].select {|x| x.include? ".conflict"} +result["conflicts"].flatten
|
991
|
+
# if all.size == 1
|
992
|
+
# num = "1 conflict"
|
993
|
+
# else
|
994
|
+
# num = "#{result["conflicts"].size} conflicts"
|
995
|
+
# end
|
996
|
+
# say "Project contains #{num}:", Thor::Shell::Color::RED
|
997
|
+
# say "#{all.join("\n")}"
|
998
|
+
# say "Please fix them, and retry", Thor::Shell::Color::RED
|
999
|
+
# exit(1)
|
1000
|
+
# end
|
961
1001
|
update_count = 0
|
962
1002
|
update_total = result["updated_on_server"].size + result["conflicts"].size
|
963
1003
|
|
@@ -1010,12 +1050,35 @@ module Cnvrg
|
|
1010
1050
|
say successful_changes.join("\n"), Thor::Shell::Color::GREEN
|
1011
1051
|
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
1012
1052
|
else
|
1013
|
-
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN
|
1053
|
+
say "#{check} Downloaded changes successfully", Thor::Shell::Color::GREEN if !sync
|
1014
1054
|
end
|
1015
1055
|
|
1016
1056
|
|
1017
1057
|
log_end(0)
|
1018
1058
|
end
|
1059
|
+
rescue
|
1060
|
+
log_end(-1)
|
1061
|
+
|
1062
|
+
say "Error occurd, \nAborting", Thor::Shell::Color::BLUE
|
1063
|
+
if successful_changes.nil?
|
1064
|
+
exit(1)
|
1065
|
+
end
|
1066
|
+
successful_changes.each do |f|
|
1067
|
+
|
1068
|
+
abs_path = "#{@project.local_path}/#{f}"
|
1069
|
+
filename = File.basename abs_path
|
1070
|
+
say "revoking #{filename}"
|
1071
|
+
if result["conflicts"].include? f
|
1072
|
+
@files.revoke_download_file(abs_path, f, filename, true)
|
1073
|
+
elsif result["updated_on_server"].include? f
|
1074
|
+
if File.directory? abs_path
|
1075
|
+
@files.revoke_download_dir(abs_path, f, project_home)
|
1076
|
+
else
|
1077
|
+
@files.revoke_download_file(project_home, abs_path, filename)
|
1078
|
+
end
|
1079
|
+
end
|
1080
|
+
end
|
1081
|
+
exit(1)
|
1019
1082
|
rescue SignalException
|
1020
1083
|
log_end(-1)
|
1021
1084
|
say "\nAborting", Thor::Shell::Color::BLUE
|
@@ -1113,9 +1176,7 @@ module Cnvrg
|
|
1113
1176
|
|
1114
1177
|
invoke :download, [], :new_branch => options["new_branch"], :verbose => options["verbose"], :sync=>true
|
1115
1178
|
invoke :upload, [], :new_branch => options["new_branch"], :verbose => options["verbose"],:sync=>true
|
1116
|
-
check = Helpers.checkmark()
|
1117
1179
|
|
1118
|
-
say "#{check} Syncing project completed successfully", Thor::Shell::Color::GREEN
|
1119
1180
|
|
1120
1181
|
end
|
1121
1182
|
|
@@ -1127,6 +1188,7 @@ module Cnvrg
|
|
1127
1188
|
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
1128
1189
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1129
1190
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
1191
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
1130
1192
|
method_option :sync_before, :type => :boolean, :aliases => ["-sb"], :default => true
|
1131
1193
|
method_option :sync_after, :type => :boolean, :aliases => ["-sa"], :default => true
|
1132
1194
|
method_option :title, :type => :string, :aliases => ["-t"], :default => ""
|
@@ -1136,6 +1198,7 @@ module Cnvrg
|
|
1136
1198
|
method_option :commit, :type => :string, :aliases => ["-c"], :default => ""
|
1137
1199
|
method_option :schedule, :type => :string, :aliases => ["-s"], :default => ""
|
1138
1200
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1201
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
1139
1202
|
|
1140
1203
|
def run(*cmd)
|
1141
1204
|
sync_before = options["sync_before"]
|
@@ -1148,15 +1211,17 @@ module Cnvrg
|
|
1148
1211
|
local = options["local"]
|
1149
1212
|
schedule = options["schedule"]
|
1150
1213
|
image = options["image"]
|
1214
|
+
grid = options["grid"]
|
1151
1215
|
if local
|
1152
1216
|
invoke :exec, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title,
|
1153
1217
|
:log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image => image
|
1154
1218
|
return
|
1155
1219
|
else
|
1156
|
-
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1220
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1221
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
1157
1222
|
instance_type = get_instance_type(instances)
|
1158
1223
|
invoke :exec_remote, [cmd], :sync_before => sync_before, :sync_after => sync_after, :title => title, :machine_type => instance_type,
|
1159
|
-
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image
|
1224
|
+
:schedule => schedule, :log => log, :email_notification => email_notification, :upload_output => upload_output, :commit => commit, :image =>image, :grid => grid
|
1160
1225
|
return
|
1161
1226
|
end
|
1162
1227
|
end
|
@@ -1315,6 +1380,8 @@ module Cnvrg
|
|
1315
1380
|
say "The process exited!", Thor::Shell::Color::RED
|
1316
1381
|
rescue => e
|
1317
1382
|
log_end(-1, e.message)
|
1383
|
+
res = @exp.end(log, 1, start_commit, cpu_average, memory_average)
|
1384
|
+
|
1318
1385
|
say "Error occurred,aborting", Thor::Shell::Color::RED
|
1319
1386
|
exit(0)
|
1320
1387
|
end
|
@@ -1327,7 +1394,7 @@ module Cnvrg
|
|
1327
1394
|
end
|
1328
1395
|
if !exp_success
|
1329
1396
|
if !Cnvrg::Helpers.internet_connection?
|
1330
|
-
wait_offline = agree "Seems like you're offline, wait until
|
1397
|
+
wait_offline = agree "Seems like you're offline, wait until you're back online?", Thor::Shell::Color::YELLOW
|
1331
1398
|
if wait_offline
|
1332
1399
|
say "Waiting until your'e online..", Thor::Shell::Color::BLUE
|
1333
1400
|
while !Cnvrg::Helpers.internet_connection?
|
@@ -1351,7 +1418,7 @@ module Cnvrg
|
|
1351
1418
|
if sync_after
|
1352
1419
|
# Sync after run
|
1353
1420
|
|
1354
|
-
download()
|
1421
|
+
download(sync=true)
|
1355
1422
|
upload(link=false, sync=true)
|
1356
1423
|
|
1357
1424
|
end
|
@@ -1363,7 +1430,6 @@ module Cnvrg
|
|
1363
1430
|
log_end(0)
|
1364
1431
|
end
|
1365
1432
|
rescue => e
|
1366
|
-
puts e
|
1367
1433
|
log_end(-1, e.message)
|
1368
1434
|
if container
|
1369
1435
|
container.stop()
|
@@ -1587,12 +1653,14 @@ module Cnvrg
|
|
1587
1653
|
method_option :schedule, :type => :string, :aliases => [ "-s"], :default => ""
|
1588
1654
|
method_option :commit, :type => :string, :aliases => [ "-c"], :default => nil
|
1589
1655
|
method_option :image, :type => :string, :aliases => ["-i"], :default => ""
|
1656
|
+
method_option :grid, :type => :string, :aliases => ["-g", "--grid"], :default => ""
|
1590
1657
|
|
1591
1658
|
def exec_remote(*cmd)
|
1592
1659
|
verify_logged_in(true)
|
1593
1660
|
log_start(__method__, args, options)
|
1594
1661
|
working_dir = is_cnvrg_dir
|
1595
1662
|
begin
|
1663
|
+
grid = options["grid"] || nil
|
1596
1664
|
|
1597
1665
|
instance_type = options["machine_type"] || nil
|
1598
1666
|
schedule = options["schedule"] || ""
|
@@ -1622,7 +1690,7 @@ module Cnvrg
|
|
1622
1690
|
upload_output_option = "--upload_output=#{upload_output}"
|
1623
1691
|
end
|
1624
1692
|
options_hash = Hash[options]
|
1625
|
-
options_hash.except!("schedule", "machine_type", "image", "upload_output")
|
1693
|
+
options_hash.except!("schedule", "machine_type", "image", "upload_output", "grid")
|
1626
1694
|
exec_options = options_hash.map { |x| "--#{x[0]}=#{x[1]}" }.flatten.join(" ")
|
1627
1695
|
command = "#{exec_options} #{upload_output_option} #{cmd.flatten.join(" ")}"
|
1628
1696
|
commit_to_run = options["commit"] || nil
|
@@ -1663,7 +1731,7 @@ module Cnvrg
|
|
1663
1731
|
say "Running remote experiment", Thor::Shell::Color::BLUE
|
1664
1732
|
exp = Experiment.new(project.owner, project.slug)
|
1665
1733
|
|
1666
|
-
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp)
|
1734
|
+
res = exp.exec_remote(command, commit_to_run, instance_type, image_slug, schedule, local_timestamp, grid)
|
1667
1735
|
if Cnvrg::CLI.is_response_success(res)
|
1668
1736
|
|
1669
1737
|
# if res["result"]["machine"] == -1
|
@@ -1697,7 +1765,13 @@ module Cnvrg
|
|
1697
1765
|
# end
|
1698
1766
|
# else
|
1699
1767
|
check = Helpers.checkmark()
|
1700
|
-
|
1768
|
+
str = "#{check} Experiment's is on: #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments/#{res["result"]["exp_url"]}"
|
1769
|
+
|
1770
|
+
if res["result"]["grid"]
|
1771
|
+
str = "Running grid search. #{Cnvrg::Helpers.remote_url}/#{project.owner}/projects/#{project.slug}/experiments?grid=#{res["result"]["exp_url"]} "
|
1772
|
+
end
|
1773
|
+
|
1774
|
+
say str, Thor::Shell::Color::GREEN
|
1701
1775
|
|
1702
1776
|
exit(0)
|
1703
1777
|
# end
|
@@ -1821,6 +1895,7 @@ module Cnvrg
|
|
1821
1895
|
method_option :large, :type => :boolean, :aliases => ["-lg","--large"], :default => false
|
1822
1896
|
method_option :gpu, :type => :boolean, :aliases => ["--gpu"], :default => false
|
1823
1897
|
method_option :gpuxl, :type => :boolean, :aliases => ["--gpuxl"], :default => false
|
1898
|
+
method_option :gpuxxl, :type => :boolean, :aliases => ["--gpuxxl"], :default => false
|
1824
1899
|
method_option :image, :type => :string, :aliases => ["--i"], :default => ""
|
1825
1900
|
|
1826
1901
|
desc 'starts a notebook session', 'starts a notebook session remotely or locally'
|
@@ -1835,7 +1910,8 @@ module Cnvrg
|
|
1835
1910
|
invoke :run_notebook, [], :notebook_dir => notebook_dir, :remote => false, :kernel => kernel,:image =>image
|
1836
1911
|
return
|
1837
1912
|
else
|
1838
|
-
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1913
|
+
instances = {"small" => options["small"], "medium" => options["medium"], "large" => options["large"],
|
1914
|
+
"gpu" => options["gpu"], "gpuxl" => options["gpuxl"], "gpuxxl"=> options["gpuxxl"]}
|
1839
1915
|
instance_type = get_instance_type(instances)
|
1840
1916
|
|
1841
1917
|
invoke :remote_notebook, [], :notebook_dir => notebook_dir, :kernel => kernel, :machine_type => instance_type, :image =>image
|
@@ -3656,6 +3732,28 @@ module Cnvrg
|
|
3656
3732
|
return new_time
|
3657
3733
|
|
3658
3734
|
end
|
3735
|
+
def update_deleted(deleted)
|
3736
|
+
final = []
|
3737
|
+
deleted.each do |d|
|
3738
|
+
all_subs = d.split("/")
|
3739
|
+
to_add = true
|
3740
|
+
value = all_subs[0]
|
3741
|
+
all_subs.each_with_index do |a,i|
|
3742
|
+
if final.include? value+"/"
|
3743
|
+
to_add = false
|
3744
|
+
break
|
3745
|
+
end
|
3746
|
+
value = value+"/"+all_subs[i+1] if i <all_subs.size-1
|
3747
|
+
end
|
3748
|
+
final << d if to_add
|
3749
|
+
|
3750
|
+
|
3751
|
+
end
|
3752
|
+
|
3753
|
+
|
3754
|
+
return final
|
3755
|
+
end
|
3756
|
+
|
3659
3757
|
|
3660
3758
|
|
3661
3759
|
end
|
data/lib/cnvrg/experiment.rb
CHANGED
@@ -78,12 +78,13 @@ module Cnvrg
|
|
78
78
|
|
79
79
|
end
|
80
80
|
|
81
|
-
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp)
|
81
|
+
def exec_remote(command, commit_to_run, instance_type, image_slug,scheduling_query,local_timestamp, grid)
|
82
82
|
response = Cnvrg::API.request("users/#{@owner}/projects/#{@project_slug}/experiment/remote", 'POST', {command: command, image_slug: image_slug,
|
83
83
|
commit_sha1: commit_to_run,
|
84
84
|
instance_type: instance_type,
|
85
85
|
scheduling_query:scheduling_query,
|
86
|
-
local_timestamp:local_timestamp
|
86
|
+
local_timestamp:local_timestamp,
|
87
|
+
grid: grid})
|
87
88
|
return response
|
88
89
|
end
|
89
90
|
|
@@ -95,10 +96,17 @@ module Cnvrg
|
|
95
96
|
end
|
96
97
|
|
97
98
|
def end(output, exit_status, end_commit, cpu_average, memory_average)
|
99
|
+
#if remote try to remove
|
98
100
|
response = Cnvrg::API.request(@base_resource + "experiment/end", 'POST', {output: output, exp_slug: @slug,
|
99
101
|
exit_status: exit_status, end_commit: end_commit,
|
100
102
|
cpu_average: cpu_average, memory_average: memory_average})
|
101
103
|
Cnvrg::CLI.is_response_success(response)
|
104
|
+
|
105
|
+
begin
|
106
|
+
FileUtils.rm_rf(["/home/ds/.cnvrg/tmp/exec.log"])
|
107
|
+
rescue
|
108
|
+
|
109
|
+
end
|
102
110
|
end
|
103
111
|
end
|
104
112
|
end
|
data/lib/cnvrg/files.rb
CHANGED
@@ -130,7 +130,6 @@ module Cnvrg
|
|
130
130
|
end
|
131
131
|
return false
|
132
132
|
rescue =>e
|
133
|
-
puts e
|
134
133
|
end
|
135
134
|
|
136
135
|
end
|
@@ -171,7 +170,6 @@ module Cnvrg
|
|
171
170
|
end
|
172
171
|
return false
|
173
172
|
rescue =>e
|
174
|
-
puts e
|
175
173
|
end
|
176
174
|
|
177
175
|
end
|
@@ -190,7 +188,6 @@ module Cnvrg
|
|
190
188
|
|
191
189
|
return true
|
192
190
|
rescue => e
|
193
|
-
puts e
|
194
191
|
return false
|
195
192
|
end
|
196
193
|
|
@@ -226,7 +223,6 @@ module Cnvrg
|
|
226
223
|
upload_file(file_path)
|
227
224
|
return resp
|
228
225
|
rescue =>e
|
229
|
-
puts e
|
230
226
|
return false
|
231
227
|
|
232
228
|
end
|
@@ -310,8 +306,6 @@ module Cnvrg
|
|
310
306
|
end
|
311
307
|
|
312
308
|
rescue =>e
|
313
|
-
puts e
|
314
|
-
puts e.backtrace
|
315
309
|
return false
|
316
310
|
|
317
311
|
end
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -72,7 +72,7 @@ module Cnvrg
|
|
72
72
|
|
73
73
|
def cnvrgignore_content
|
74
74
|
%{
|
75
|
-
|
75
|
+
# cnvrg ignore: Ignore the following directories and files
|
76
76
|
# for example:
|
77
77
|
# some_dir/
|
78
78
|
# some_file.txt
|
@@ -81,6 +81,47 @@ module Cnvrg
|
|
81
81
|
}.strip
|
82
82
|
end
|
83
83
|
|
84
|
+
def hyper_content
|
85
|
+
%{# Hyperparameter Optimization is the process of choosing a set of parameters for a learning algorithm, usually with the goal of optimizing a measure of the algorithm's performance on an independent data set.
|
86
|
+
|
87
|
+
# Below is the list of parameters that will be used in the optimization process. Each parameter has a param_name that should match the argument that is feeded to the experiment s.t kernel => --kernel='rbf'
|
88
|
+
|
89
|
+
parameters:
|
90
|
+
# Integer parameter is a range of possible values between a minimum (inclusive)
|
91
|
+
# and maximum (not inclusive) values. Values are floored (0.7 => 0)
|
92
|
+
- param_name: "learning_rate"
|
93
|
+
type: "integer"
|
94
|
+
min: 0 # inclusive
|
95
|
+
max: 10 # not inclusive
|
96
|
+
scale: "linear"
|
97
|
+
steps: 4 # The number of linear steps to produce.
|
98
|
+
|
99
|
+
|
100
|
+
# Float parameter is a range of possible values between a minimum (inclusive)
|
101
|
+
# and maximum (not inclusive) values.
|
102
|
+
#
|
103
|
+
- param_name: "learning_rate"
|
104
|
+
type: "float" # precision is 9 after period
|
105
|
+
min: 0.00001
|
106
|
+
max: 0.1
|
107
|
+
scale: "log2" # Could be log10 as well
|
108
|
+
steps: 2
|
109
|
+
|
110
|
+
# Discrete parameter is an array of numerical values.
|
111
|
+
#
|
112
|
+
- param_name: "c"
|
113
|
+
type: "discrete"
|
114
|
+
values: [0, 0.1 ,0.001]
|
115
|
+
|
116
|
+
# Categorical parameter is an array of string values
|
117
|
+
#
|
118
|
+
- param_name: "kernel"
|
119
|
+
type: "categorical"
|
120
|
+
values: ["linear", "poly", "rbf"]
|
121
|
+
|
122
|
+
}
|
123
|
+
end
|
124
|
+
|
84
125
|
def readme_content
|
85
126
|
%{
|
86
127
|
# README
|
data/lib/cnvrg/job.rb
CHANGED
data/lib/cnvrg/project.rb
CHANGED
@@ -90,6 +90,7 @@ module Cnvrg
|
|
90
90
|
]
|
91
91
|
cnvrgreadme = Helpers.readme_content
|
92
92
|
cnvrgignore = Helpers.cnvrgignore_content
|
93
|
+
cnvrghyper = Helpers.hyper_content
|
93
94
|
|
94
95
|
begin
|
95
96
|
|
@@ -109,6 +110,8 @@ module Cnvrg
|
|
109
110
|
File.open(project_name + "/.cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
110
111
|
File.open(project_name + "/.cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
111
112
|
File.open(project_name + "/README.md", "w+") { |f| f.write cnvrgreadme }
|
113
|
+
File.open(project_name + "/src/hyper.yaml", "w+") { |f| f.write cnvrghyper }
|
114
|
+
|
112
115
|
rescue
|
113
116
|
return false
|
114
117
|
end
|
@@ -137,14 +140,13 @@ module Cnvrg
|
|
137
140
|
FileUtils.mkdir_p list_dirs
|
138
141
|
FileUtils.touch list_files
|
139
142
|
File.open(".cnvrg/config.yml", "w+") { |f| f.write config.to_yaml }
|
140
|
-
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore }
|
143
|
+
File.open(".cnvrgignore", "w+") { |f| f.write cnvrgignore } unless File.exist? ".cnvrgignore"
|
141
144
|
if !File.exist? "README" and !File.exist? "README.md"
|
142
145
|
FileUtils.touch [ "README.md" ]
|
143
146
|
File.open("README.md", "w+") { |f| f.write cnvrgreadme }
|
144
147
|
end
|
145
148
|
|
146
149
|
rescue => e
|
147
|
-
puts e
|
148
150
|
return false
|
149
151
|
end
|
150
152
|
return true
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.145
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-04-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -249,72 +249,72 @@ dependencies:
|
|
249
249
|
name: logstash-logger
|
250
250
|
requirement: !ruby/object:Gem::Requirement
|
251
251
|
requirements:
|
252
|
-
- - "
|
252
|
+
- - "~>"
|
253
253
|
- !ruby/object:Gem::Version
|
254
|
-
version:
|
254
|
+
version: 0.22.1
|
255
255
|
type: :runtime
|
256
256
|
prerelease: false
|
257
257
|
version_requirements: !ruby/object:Gem::Requirement
|
258
258
|
requirements:
|
259
|
-
- - "
|
259
|
+
- - "~>"
|
260
260
|
- !ruby/object:Gem::Version
|
261
|
-
version:
|
261
|
+
version: 0.22.1
|
262
262
|
- !ruby/object:Gem::Dependency
|
263
263
|
name: launchy
|
264
264
|
requirement: !ruby/object:Gem::Requirement
|
265
265
|
requirements:
|
266
|
-
- - "
|
266
|
+
- - "~>"
|
267
267
|
- !ruby/object:Gem::Version
|
268
|
-
version: '
|
268
|
+
version: '2.4'
|
269
269
|
type: :runtime
|
270
270
|
prerelease: false
|
271
271
|
version_requirements: !ruby/object:Gem::Requirement
|
272
272
|
requirements:
|
273
|
-
- - "
|
273
|
+
- - "~>"
|
274
274
|
- !ruby/object:Gem::Version
|
275
|
-
version: '
|
275
|
+
version: '2.4'
|
276
276
|
- !ruby/object:Gem::Dependency
|
277
277
|
name: docker-api
|
278
278
|
requirement: !ruby/object:Gem::Requirement
|
279
279
|
requirements:
|
280
|
-
- - "
|
280
|
+
- - "~>"
|
281
281
|
- !ruby/object:Gem::Version
|
282
|
-
version: '
|
282
|
+
version: '1.33'
|
283
283
|
type: :runtime
|
284
284
|
prerelease: false
|
285
285
|
version_requirements: !ruby/object:Gem::Requirement
|
286
286
|
requirements:
|
287
|
-
- - "
|
287
|
+
- - "~>"
|
288
288
|
- !ruby/object:Gem::Version
|
289
|
-
version: '
|
289
|
+
version: '1.33'
|
290
290
|
- !ruby/object:Gem::Dependency
|
291
291
|
name: rubyzip
|
292
292
|
requirement: !ruby/object:Gem::Requirement
|
293
293
|
requirements:
|
294
|
-
- - "
|
294
|
+
- - "~>"
|
295
295
|
- !ruby/object:Gem::Version
|
296
|
-
version: '
|
296
|
+
version: '1.2'
|
297
297
|
type: :runtime
|
298
298
|
prerelease: false
|
299
299
|
version_requirements: !ruby/object:Gem::Requirement
|
300
300
|
requirements:
|
301
|
-
- - "
|
301
|
+
- - "~>"
|
302
302
|
- !ruby/object:Gem::Version
|
303
|
-
version: '
|
303
|
+
version: '1.2'
|
304
304
|
- !ruby/object:Gem::Dependency
|
305
305
|
name: activesupport
|
306
306
|
requirement: !ruby/object:Gem::Requirement
|
307
307
|
requirements:
|
308
|
-
- - "
|
308
|
+
- - "~>"
|
309
309
|
- !ruby/object:Gem::Version
|
310
|
-
version: '0'
|
310
|
+
version: '5.0'
|
311
311
|
type: :runtime
|
312
312
|
prerelease: false
|
313
313
|
version_requirements: !ruby/object:Gem::Requirement
|
314
314
|
requirements:
|
315
|
-
- - "
|
315
|
+
- - "~>"
|
316
316
|
- !ruby/object:Gem::Version
|
317
|
-
version: '0'
|
317
|
+
version: '5.0'
|
318
318
|
description: A CLI tool for interacting with cnvrg.io.
|
319
319
|
email:
|
320
320
|
- info@cnvrg.io
|
@@ -356,8 +356,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
356
356
|
version: '0'
|
357
357
|
requirements: []
|
358
358
|
rubyforge_project:
|
359
|
-
rubygems_version: 2.
|
359
|
+
rubygems_version: 2.5.1
|
360
360
|
signing_key:
|
361
361
|
specification_version: 4
|
362
362
|
summary: A CLI tool for interacting with cnvrg.io.
|
363
363
|
test_files: []
|
364
|
+
has_rdoc:
|