cnvrg 0.0.7 → 0.0.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/api.rb +5 -1
- data/lib/cnvrg/cli.rb +80 -20
- data/lib/cnvrg/files.rb +19 -4
- data/lib/cnvrg/helpers.rb +17 -1
- data/lib/cnvrg/project.rb +2 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f68e73ce51d096ab10bae5dd5ba07b6444c4d898
|
4
|
+
data.tar.gz: a50d1f1e96a4e21d9a408d7c5c5e915888480ec2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e1d21cda34620db5df089a2e4db43b730c0a3eb5265d423101cd91d5e5c606e56b15be8df027bc7e4b0ff35eb1de9eddcb7ea6f4e735792fb3992e7d4f5f10b8
|
7
|
+
data.tar.gz: 9e8094bb8658f213aa7af9c943e3015e38fbb7d244176da607186cefb8fd64b5221d3a7d4ce27427aba45dfe8f33fed031f5e044a165e17a7625b34dcb106084
|
data/lib/cnvrg/api.rb
CHANGED
@@ -44,7 +44,7 @@ module Cnvrg
|
|
44
44
|
end
|
45
45
|
|
46
46
|
@user, @pass = n[Cnvrg::Helpers.netrc_domain]
|
47
|
-
|
47
|
+
begin
|
48
48
|
conn = Faraday.new
|
49
49
|
conn.headers['Auth-Token'] = @pass
|
50
50
|
conn.headers['User-Agent'] = "#{Cnvrg::API::USER_AGENT}"
|
@@ -102,6 +102,10 @@ module Cnvrg
|
|
102
102
|
end
|
103
103
|
else
|
104
104
|
end
|
105
|
+
rescue
|
106
|
+
return nil
|
107
|
+
end
|
108
|
+
|
105
109
|
end
|
106
110
|
|
107
111
|
def self.endpoint_uri
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -412,16 +412,16 @@ module Cnvrg
|
|
412
412
|
begin
|
413
413
|
(result["added"] + result["updated_on_local"]).each do |f|
|
414
414
|
puts f
|
415
|
+
absolute_path = "#{@project.local_path}/#{f}"
|
415
416
|
relative_path = f.gsub(/^#{@project.local_path + "/"}/, "")
|
416
|
-
|
417
|
-
|
418
|
-
resDir = @files.create_dir(f, relative_path, commit_sha1)
|
417
|
+
if File.directory?(absolute_path)
|
418
|
+
resDir = @files.create_dir(absolute_path, relative_path, commit_sha1)
|
419
419
|
if resDir
|
420
420
|
update_count += 1
|
421
421
|
successful_updates<< relative_path
|
422
422
|
end
|
423
423
|
else
|
424
|
-
res = @files.upload_file(
|
424
|
+
res = @files.upload_file(absolute_path, relative_path, commit_sha1)
|
425
425
|
if res
|
426
426
|
update_count += 1
|
427
427
|
successful_updates<< relative_path
|
@@ -453,7 +453,8 @@ module Cnvrg
|
|
453
453
|
@files.rollback_commit(commit_sha1)
|
454
454
|
say "User aborted, Rolling Back all changes.", Thor::Shell::Color::RED
|
455
455
|
exit(0)
|
456
|
-
rescue
|
456
|
+
rescue => e
|
457
|
+
puts e
|
457
458
|
@files.rollback_commit(commit_sha1)
|
458
459
|
say "Exception while trying to upload, Rolling back", Thor::Shell::Color::RED
|
459
460
|
exit(0)
|
@@ -493,7 +494,9 @@ module Cnvrg
|
|
493
494
|
@files.rollback_commit(commit_sha1)
|
494
495
|
end
|
495
496
|
rescue SignalException
|
496
|
-
say "
|
497
|
+
say "\nAborting",Thor::Shell::Color::BLUE
|
498
|
+
say "\nRolling back all changes",Thor::Shell::Color::BLUE
|
499
|
+
@files.rollback_commit(commit_sha1)
|
497
500
|
exit(1)
|
498
501
|
end
|
499
502
|
|
@@ -539,6 +542,7 @@ module Cnvrg
|
|
539
542
|
# dir
|
540
543
|
if @files.download_dir(f, relative_path, project_home)
|
541
544
|
successful_changes << relative_path
|
545
|
+
|
542
546
|
end
|
543
547
|
else
|
544
548
|
# blob
|
@@ -556,7 +560,25 @@ module Cnvrg
|
|
556
560
|
say "Total of #{successful_changes.size} / #{update_total} files.", Thor::Shell::Color::GREEN
|
557
561
|
end
|
558
562
|
rescue SignalException
|
559
|
-
say "
|
563
|
+
say "\nAborting", Thor::Shell::Color::BLUE
|
564
|
+
if successful_changes.nil?
|
565
|
+
exit(1)
|
566
|
+
end
|
567
|
+
successful_changes.each do |f|
|
568
|
+
|
569
|
+
abs_path = "#{@project.local_path}/#{f}"
|
570
|
+
filename = File.basename abs_path
|
571
|
+
say "revoking #{filename}"
|
572
|
+
if result["conflicts"].include? f
|
573
|
+
@files.revoke_download_file(abs_path,f,filename,true)
|
574
|
+
elsif result["updated_on_server"].include? f
|
575
|
+
if File.directory? abs_path
|
576
|
+
@files.revoke_download_dir(abs_path,f,project_home)
|
577
|
+
else
|
578
|
+
@files.revoke_download_file(project_home,abs_path,filename)
|
579
|
+
end
|
580
|
+
end
|
581
|
+
end
|
560
582
|
exit(1)
|
561
583
|
end
|
562
584
|
end
|
@@ -602,7 +624,9 @@ module Cnvrg
|
|
602
624
|
method_option :upload_output, :type => :string, :aliases => ["--uo", "-uo"], :default => ""
|
603
625
|
|
604
626
|
def exec(*cmd)
|
605
|
-
|
627
|
+
log = []
|
628
|
+
cpu_average =0
|
629
|
+
memory_average = 0
|
606
630
|
verify_logged_in()
|
607
631
|
|
608
632
|
project_home = get_project_home
|
@@ -614,11 +638,12 @@ module Cnvrg
|
|
614
638
|
email_notification = options["email_notification"]
|
615
639
|
upload_output = options["upload_output"]
|
616
640
|
time_to_upload = calc_output_time(upload_output)
|
617
|
-
|
641
|
+
begin
|
618
642
|
if sync_before
|
619
643
|
# Sync before run
|
620
644
|
say "Syncing project before running", Thor::Shell::Color::BLUE
|
621
645
|
say 'Checking for new updates from remote version', Thor::Shell::Color::BLUE
|
646
|
+
|
622
647
|
download()
|
623
648
|
upload()
|
624
649
|
say "Done Syncing", Thor::Shell::Color::BLUE
|
@@ -626,7 +651,7 @@ module Cnvrg
|
|
626
651
|
|
627
652
|
start_commit = @project.last_local_commit
|
628
653
|
cmd = cmd.join("\s")
|
629
|
-
|
654
|
+
|
630
655
|
say "Running: #{cmd}\n", Thor::Shell::Color::BLUE
|
631
656
|
|
632
657
|
@exp = Experiment.new(@project.owner, @project.slug)
|
@@ -698,28 +723,42 @@ module Cnvrg
|
|
698
723
|
rescue Errno::ENOENT
|
699
724
|
|
700
725
|
exp_success = false
|
726
|
+
|
701
727
|
say "command \"#{cmd}\" couldn't be executed, verify command is valid", Thor::Shell::Color::RED
|
702
728
|
rescue PTY::ChildExited
|
703
729
|
|
704
730
|
exp_success = false
|
705
731
|
say "The process exited!", Thor::Shell::Color::RED
|
706
732
|
rescue => e
|
707
|
-
ensure
|
708
|
-
::Process.wait pid
|
709
733
|
|
710
734
|
end
|
735
|
+
::Process.wait pid
|
711
736
|
cpu_average = cpu_total.inject(0) { |sum, el| sum + el }.to_f / cpu_total.size
|
712
737
|
memory_average = memory_total.inject(0) { |sum, el| sum + el }.to_f / memory_total.size
|
713
738
|
exit_status = $?.exitstatus
|
714
|
-
|
715
|
-
if exit_status != 0
|
739
|
+
if $?.exitstatus != 0
|
716
740
|
exp_success = false
|
717
741
|
end
|
718
742
|
if !exp_success
|
743
|
+
if !Cnvrg::Helpers.internet_connection?
|
744
|
+
wait_offline = agree "Seems like you're offline, wait until your'e back online?",limited_to: ['y', 'n'], default: 'n'
|
745
|
+
if wait_offline
|
746
|
+
say "Waiting until your'e online..", Thor::Shell::Color::BLUE
|
747
|
+
while !Cnvrg::Helpers.internet_connection?
|
748
|
+
end
|
749
|
+
exit_status = 0
|
750
|
+
else
|
751
|
+
say "Experiment has failed, your'e computer is offline", Thor::Shell::Color::RED
|
752
|
+
exit(0)
|
753
|
+
end
|
754
|
+
else
|
755
|
+
|
719
756
|
end_commit = @project.last_local_commit
|
720
757
|
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
721
758
|
say "Experiment has failed, look at the log for more details or run cnvrg exec --log", Thor::Shell::Color::RED
|
722
759
|
exit(0)
|
760
|
+
end
|
761
|
+
|
723
762
|
end
|
724
763
|
if sync_after
|
725
764
|
say "Syncing project after running", Thor::Shell::Color::BLUE
|
@@ -735,6 +774,7 @@ module Cnvrg
|
|
735
774
|
say "#{check} Done. Experiment's result: #{Cnvrg::Helpers.remote_url}/#{@project.owner}/projects/#{@project.slug}/experiments/#{@exp.slug}", Thor::Shell::Color::GREEN
|
736
775
|
end
|
737
776
|
rescue
|
777
|
+
|
738
778
|
say "Couldn't run #{cmd}, check your input parameters", Thor::Shell::Color::RED
|
739
779
|
exit(1)
|
740
780
|
end
|
@@ -744,7 +784,12 @@ module Cnvrg
|
|
744
784
|
|
745
785
|
end
|
746
786
|
rescue SignalException
|
747
|
-
|
787
|
+
exit_status = -1
|
788
|
+
end_commit = @project.last_local_commit
|
789
|
+
|
790
|
+
res = @exp.end(log, exit_status, end_commit, cpu_average, memory_average)
|
791
|
+
say "\nAborting"
|
792
|
+
|
748
793
|
exit(1)
|
749
794
|
end
|
750
795
|
end
|
@@ -802,7 +847,7 @@ module Cnvrg
|
|
802
847
|
elsif run_docker.include? "port is already allocated"
|
803
848
|
say "Couldn't start notebook with port 8888, port is taken", Thor::Shell::Color::RED
|
804
849
|
port_container = `#{docker_path} ps |grep 8888 |awk '{print $1}'`.strip!
|
805
|
-
stop =
|
850
|
+
stop = agree "There is another running notebook server: #{port_container}, do you want to stop it?", limited_to: ['y', 'n'], default: 'y'
|
806
851
|
if stop == "y"
|
807
852
|
did_stop = system("#{docker_path} stop #{port_container}")
|
808
853
|
if !did_stop
|
@@ -1114,12 +1159,19 @@ module Cnvrg
|
|
1114
1159
|
end
|
1115
1160
|
|
1116
1161
|
def self.is_response_success(response, should_exit=true)
|
1117
|
-
if response
|
1162
|
+
if response.nil?
|
1163
|
+
if Cnvrg::Helpers.internet_connection?
|
1164
|
+
say("<%= color('Error:You seems to be offline', RED) %>")
|
1165
|
+
else
|
1166
|
+
say("<%= color('Error', RED) %>")
|
1167
|
+
end
|
1168
|
+
elsif response["status"]!= 200
|
1118
1169
|
error = response['message']
|
1119
1170
|
if response["status"] == 500
|
1120
1171
|
say("<%= color('Server Error', RED) %>")
|
1121
1172
|
else
|
1122
|
-
|
1173
|
+
puts response
|
1174
|
+
say("<%= color('Error:#{error}', RED) %>")
|
1123
1175
|
end
|
1124
1176
|
|
1125
1177
|
if should_exit
|
@@ -1177,9 +1229,17 @@ module Cnvrg
|
|
1177
1229
|
say 'Please log in via `cnvrg login`', Thor::Shell::Color::YELLOW
|
1178
1230
|
exit(1)
|
1179
1231
|
end
|
1232
|
+
|
1180
1233
|
if !Helpers.internet_connection?
|
1181
|
-
|
1182
|
-
|
1234
|
+
wait_offline = agree "Seems like you're offline, wait until your'e back online?",limited_to: ['y', 'n'], default: 'n'
|
1235
|
+
if wait_offline
|
1236
|
+
say "Waiting until your'e online..", Thor::Shell::Color::BLUE
|
1237
|
+
while !Cnvrg::Helpers.internet_connection?
|
1238
|
+
end
|
1239
|
+
else
|
1240
|
+
say "you seem to be offline, please check your internet connection", Thor::Shell::Color::RED
|
1241
|
+
exit(0)
|
1242
|
+
end
|
1183
1243
|
end
|
1184
1244
|
|
1185
1245
|
config = YAML.load_file(File.expand_path('~')+"/.cnvrg/config.yml")
|
data/lib/cnvrg/files.rb
CHANGED
@@ -17,18 +17,18 @@ module Cnvrg
|
|
17
17
|
|
18
18
|
def upload_file(absolute_path, relative_path, commit_sha1)
|
19
19
|
file_name = File.basename relative_path
|
20
|
-
file_size = File.size(
|
21
|
-
mime_type = MimeMagic.by_path(
|
20
|
+
file_size = File.size(absolute_path).to_f
|
21
|
+
mime_type = MimeMagic.by_path(absolute_path)
|
22
22
|
content_type = !(mime_type.nil? or mime_type.text?) ? mime_type.type : "text/plain"
|
23
23
|
upload_resp = Cnvrg::API.request(@base_resource + "upload_file", 'POST_FILE', {absolute_path: absolute_path, relative_path: relative_path,
|
24
24
|
commit_sha1: commit_sha1, file_name: file_name,
|
25
25
|
file_size: file_size, file_content_type: content_type})
|
26
26
|
if Cnvrg::CLI.is_response_success(upload_resp, false)
|
27
27
|
if file_size.to_f>= Cnvrg::Files::LARGE_FILE.to_f
|
28
|
-
s3_res = upload_large_files_s3(upload_resp,
|
28
|
+
s3_res = upload_large_files_s3(upload_resp, absolute_path)
|
29
29
|
else
|
30
30
|
path = upload_resp["result"]["path"]
|
31
|
-
s3_res = upload_small_files_s3(path,
|
31
|
+
s3_res = upload_small_files_s3(path, absolute_path, content_type)
|
32
32
|
end
|
33
33
|
|
34
34
|
if s3_res
|
@@ -186,6 +186,21 @@ module Cnvrg
|
|
186
186
|
def download_dir(absolute_path, relative_path, project_home)
|
187
187
|
FileUtils.mkdir_p("#{project_home}/#{absolute_path}")
|
188
188
|
end
|
189
|
+
def revoke_download_dir(absolute_path, relative_path, project_home)
|
190
|
+
puts FileUtils.rmtree("#{absolute_path}")
|
191
|
+
end
|
192
|
+
|
193
|
+
def revoke_download_file(project_home,absolute_path,filename,conflict=false)
|
194
|
+
begin
|
195
|
+
file_location = absolute_path.gsub(/#{filename}\/?$/, "")
|
196
|
+
|
197
|
+
filename += ".conflict" if conflict
|
198
|
+
FileUtils.remove("#{file_location}/#{filename}")
|
199
|
+
return true
|
200
|
+
rescue
|
201
|
+
return false
|
202
|
+
end
|
203
|
+
end
|
189
204
|
|
190
205
|
def start_commit
|
191
206
|
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -15,7 +15,23 @@ module Cnvrg
|
|
15
15
|
end
|
16
16
|
end
|
17
17
|
def remote_url
|
18
|
-
|
18
|
+
home_dir = File.expand_path('~')
|
19
|
+
config = ""
|
20
|
+
begin
|
21
|
+
if File.exist? home_dir+"/.cnvrg/config.yml"
|
22
|
+
config = YAML.load_file(home_dir+"/.cnvrg/config.yml")
|
23
|
+
else
|
24
|
+
return "https://cnvrg.io"
|
25
|
+
end
|
26
|
+
|
27
|
+
rescue
|
28
|
+
return "https://cnvrg.io"
|
29
|
+
end
|
30
|
+
if !config or config.empty? or config.to_h[:api].nil?
|
31
|
+
return "https://cnvrg.io"
|
32
|
+
else
|
33
|
+
return config.to_h[:api].gsub("/api","")
|
34
|
+
end
|
19
35
|
end
|
20
36
|
|
21
37
|
def windows?
|
data/lib/cnvrg/project.rb
CHANGED
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-01-
|
12
|
+
date: 2017-01-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|