cnvrg 1.2.6 → 1.2.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/cnvrg/cli.rb +32 -7
- data/lib/cnvrg/datafiles.rb +1 -1
- data/lib/cnvrg/dataset.rb +1 -1
- data/lib/cnvrg/helpers.rb +8 -2
- data/lib/cnvrg/project.rb +48 -1
- data/lib/cnvrg/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f7319668db1dc7e8257aa9f6067fc43b91dd559f4e73c1c5d84c6e0a6c4ec30b
|
4
|
+
data.tar.gz: 3958b12ea50a32fa38ebc0c41a3e0b019199fe3f6a4f0f3ebcbe43b839fa25c1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb413f91c9968b93ec5e1b05be0a2deff6f5aaa5e097b9eda6e02311a36e6b14d8dba34a8385c0e1abc5193810fbaa1a0902f49537ffb79e2a7745117448233c
|
7
|
+
data.tar.gz: 5a4d02fc1920d0fda7e0d5b7dfab5793ee31e6adc8b9fc2e60adc5e55213284dcc3bb3cb1c1974b1da0541da9fd8a86eb94d4c3f1c3f0a82543ec43fd4dcd080
|
data/lib/cnvrg/cli.rb
CHANGED
@@ -150,7 +150,7 @@ module Cnvrg
|
|
150
150
|
IP = "localhost"
|
151
151
|
PORT = 7654
|
152
152
|
|
153
|
-
ParallelThreads ||=
|
153
|
+
ParallelThreads ||= Cnvrg::Helpers.parallel_threads
|
154
154
|
ParallelProcesses ||= Parallel.processor_count
|
155
155
|
|
156
156
|
class << self
|
@@ -451,19 +451,44 @@ module Cnvrg
|
|
451
451
|
|
452
452
|
desc 'auth', "hidden..", hide: true
|
453
453
|
def auth
|
454
|
-
|
455
|
-
owner = ENV['CNVRG_OWNER']
|
456
|
-
user = ENV['CNVRG_USER']
|
457
|
-
api = ENV['CNVRG_API']
|
458
|
-
|
454
|
+
token = ENV['CNVRG_TOKEN'] || exit(1)
|
455
|
+
owner = ENV['CNVRG_OWNER'] || exit(1)
|
456
|
+
user = ENV['CNVRG_USER'] || exit(1)
|
457
|
+
api = ENV['CNVRG_API'] || exit(1)
|
458
|
+
email = ENV['CNVRG_EMAIL'] || exit(1)
|
459
459
|
|
460
460
|
netrc = Netrc.read
|
461
|
-
netrc[Cnvrg::Helpers.netrc_domain] =
|
461
|
+
netrc[Cnvrg::Helpers.netrc_domain] = email, token
|
462
462
|
netrc.save
|
463
463
|
|
464
464
|
set_owner(owner, user, api)
|
465
465
|
end
|
466
466
|
|
467
|
+
desc 'check_spot', "hidden..", hide: true
|
468
|
+
def check_spot
|
469
|
+
log_start(__method__, args, options)
|
470
|
+
project_home = get_project_home
|
471
|
+
@project = Project.new(project_home)
|
472
|
+
|
473
|
+
log_message('Checking Spot Instance Status', Thor::Shell::Color::YELLOW)
|
474
|
+
will_terminate = @project.spot_will_terminate
|
475
|
+
if not will_terminate
|
476
|
+
log_message("Not spot termination detected", Thor::Shell::Color::YELLOW)
|
477
|
+
return
|
478
|
+
end
|
479
|
+
job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
|
480
|
+
machine_activity = @project.get_machine_activity
|
481
|
+
|
482
|
+
sync_force = job_type == "NotebookSession" ? false : true
|
483
|
+
upload(false, false, true, '', true, sync_force , ENV['CNVRG_OUTPUT_DIR'], job_type, job_id)
|
484
|
+
log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
|
485
|
+
res = nil
|
486
|
+
while res.blank?
|
487
|
+
res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
|
488
|
+
sleep(10)
|
489
|
+
end
|
490
|
+
end
|
491
|
+
|
467
492
|
|
468
493
|
desc 'login', 'Authenticate with cnvrg.io platform'
|
469
494
|
|
data/lib/cnvrg/datafiles.rb
CHANGED
data/lib/cnvrg/dataset.rb
CHANGED
@@ -4,7 +4,7 @@ module Cnvrg
|
|
4
4
|
attr_reader :slug, :owner, :title, :local_path, :working_dir
|
5
5
|
|
6
6
|
RemoteURL ||= "https://cnvrg.io"
|
7
|
-
IDXParallelThreads ||=
|
7
|
+
IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
|
8
8
|
IDXParallelProcesses ||= Parallel.processor_count
|
9
9
|
|
10
10
|
def initialize(project_home = '', dataset_url: '')
|
data/lib/cnvrg/helpers.rb
CHANGED
@@ -2,9 +2,15 @@ module Cnvrg
|
|
2
2
|
module Helpers
|
3
3
|
|
4
4
|
extend self
|
5
|
-
ParallelThreads ||= 20
|
6
5
|
def parallel_threads()
|
7
|
-
|
6
|
+
threads = ENV["CNVRG_PARALLEL_THREADS"].to_i
|
7
|
+
|
8
|
+
|
9
|
+
### if empty, default will be 15 threads
|
10
|
+
threads = threads > 0 ? threads : 15
|
11
|
+
|
12
|
+
### set max threads to be 100k
|
13
|
+
[threads, 100000].min
|
8
14
|
end
|
9
15
|
def checkmark
|
10
16
|
checkmark = "\u2713"
|
data/lib/cnvrg/project.rb
CHANGED
@@ -4,7 +4,7 @@ module Cnvrg
|
|
4
4
|
attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
|
5
5
|
|
6
6
|
RemoteURL ||= "https://cnvrg.io"
|
7
|
-
IDXParallelThreads ||=
|
7
|
+
IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
|
8
8
|
|
9
9
|
|
10
10
|
def initialize(project_home)
|
@@ -632,5 +632,52 @@ module Cnvrg
|
|
632
632
|
end
|
633
633
|
|
634
634
|
|
635
|
+
def spot_will_terminate
|
636
|
+
restart = false
|
637
|
+
config = self.get_config
|
638
|
+
# logic that will prevent multiple restart calls
|
639
|
+
if config[:spot_taken]
|
640
|
+
return false
|
641
|
+
end
|
642
|
+
begin
|
643
|
+
url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
|
644
|
+
req = Net::HTTP::Get.new(url.to_s)
|
645
|
+
res = Net::HTTP.start(url.host, url.port) {|http|
|
646
|
+
http.request(req)
|
647
|
+
}
|
648
|
+
unless res.body.include? "404"
|
649
|
+
restart = true
|
650
|
+
end
|
651
|
+
if res.body.include? "Empty reply from server"
|
652
|
+
restart = false
|
653
|
+
end
|
654
|
+
rescue
|
655
|
+
restart = false
|
656
|
+
end
|
657
|
+
|
658
|
+
if restart
|
659
|
+
config[:spot_taken] = true
|
660
|
+
self.set_config(config)
|
661
|
+
end
|
662
|
+
|
663
|
+
return restart
|
664
|
+
end
|
665
|
+
|
666
|
+
def send_restart_request(job_id: nil, job_type: nil, ma_id: nil)
|
667
|
+
Cnvrg::API.request("#{base_resource}/spot_restart", 'POST', {job_type: job_type, job_id: job_id, machine_activity: ma_id})
|
668
|
+
end
|
669
|
+
|
670
|
+
def get_machine_activity
|
671
|
+
begin
|
672
|
+
machine_activity = File.open("#{@local_path}/.cnvrg/machine_activity", "rb").read
|
673
|
+
machine_activity = machine_activity.to_s.strip
|
674
|
+
ma_id = machine_activity.to_i
|
675
|
+
return ma_id
|
676
|
+
rescue
|
677
|
+
return nil
|
678
|
+
end
|
679
|
+
end
|
680
|
+
|
681
|
+
|
635
682
|
end
|
636
683
|
end
|
data/lib/cnvrg/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cnvrg
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Yochay Ettun
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-03-
|
12
|
+
date: 2019-03-26 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|