cnvrg 1.2.6 → 1.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8e33d716e3fce0ef29d6684c29dec59f6edd04173d0330343d0510ec11bd087a
-  data.tar.gz: 9a019b36e6611ffd4b627c7fb9b9edecbde44be360f26034b668c52da804da8d
+  metadata.gz: f7319668db1dc7e8257aa9f6067fc43b91dd559f4e73c1c5d84c6e0a6c4ec30b
+  data.tar.gz: 3958b12ea50a32fa38ebc0c41a3e0b019199fe3f6a4f0f3ebcbe43b839fa25c1
 SHA512:
-  metadata.gz: ad76658b08c85dcab2cade07327e5e6c7f4843e9eecbc6d2c768d4e3d166723e2f007d8260d83fd6d2f3cabbb25daf85bf7b247fe7b0bd45445f8af7cae2b141
-  data.tar.gz: 22da11b1b04ec3e55fbb9a3586651783411daa1fc1c5d06548890f71770b0e152740589b883394f84172f3e5d0f0519001abaa4416031bfa0856cd553be685c3
+  metadata.gz: cb413f91c9968b93ec5e1b05be0a2deff6f5aaa5e097b9eda6e02311a36e6b14d8dba34a8385c0e1abc5193810fbaa1a0902f49537ffb79e2a7745117448233c
+  data.tar.gz: 5a4d02fc1920d0fda7e0d5b7dfab5793ee31e6adc8b9fc2e60adc5e55213284dcc3bb3cb1c1974b1da0541da9fd8a86eb94d4c3f1c3f0a82543ec43fd4dcd080

data/lib/cnvrg/cli.rb

@@ -150,7 +150,7 @@ module Cnvrg
     IP = "localhost"
     PORT = 7654
 
-    ParallelThreads ||= 15
+    ParallelThreads ||= Cnvrg::Helpers.parallel_threads
     ParallelProcesses ||= Parallel.processor_count
 
     class << self
@@ -451,19 +451,44 @@ module Cnvrg
 
     desc 'auth', "hidden..", hide: true
     def auth
-      @token = ENV['CNVRG_TOKEN']
-      owner = ENV['CNVRG_OWNER']
-      user = ENV['CNVRG_USER']
-      api = ENV['CNVRG_API']
-      @email = ENV['CNVRG_EMAIL']
+      token = ENV['CNVRG_TOKEN'] || exit(1)
+      owner = ENV['CNVRG_OWNER'] || exit(1)
+      user = ENV['CNVRG_USER'] || exit(1)
+      api = ENV['CNVRG_API'] || exit(1)
+      email = ENV['CNVRG_EMAIL'] || exit(1)
 
       netrc = Netrc.read
-      netrc[Cnvrg::Helpers.netrc_domain] = @email, @token
+      netrc[Cnvrg::Helpers.netrc_domain] = email, token
       netrc.save
 
       set_owner(owner, user, api)
     end
 
+    desc 'check_spot', "hidden..", hide: true
+    def check_spot
+      log_start(__method__, args, options)
+      project_home = get_project_home
+      @project = Project.new(project_home)
+
+      log_message('Checking Spot Instance Status', Thor::Shell::Color::YELLOW)
+      will_terminate = @project.spot_will_terminate
+      if not will_terminate
+        log_message("Not spot termination detected", Thor::Shell::Color::YELLOW)
+        return
+      end
+      job_type, job_id = ENV['CNVRG_JOB_TYPE'], ENV['CNVRG_JOB_ID']
+      machine_activity = @project.get_machine_activity
+
+      sync_force = job_type == "NotebookSession" ? false : true
+      upload(false, false, true, '', true, sync_force , ENV['CNVRG_OUTPUT_DIR'], job_type, job_id)
+      log_message('Spot instance is going to be terminated', Thor::Shell::Color::YELLOW)
+      res = nil
+      while res.blank?
+        res = @project.send_restart_request(job_type: job_type, job_id: job_id, ma_id: machine_activity)
+        sleep(10)
+      end
+    end
+
 
     desc 'login', 'Authenticate with cnvrg.io platform'
 

data/lib/cnvrg/datafiles.rb

@@ -7,7 +7,7 @@ require 'fileutils'
 
 module Cnvrg
   class Datafiles
-    ParallelThreads ||= 15
+    ParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
     LARGE_FILE=1024*1024*5
     MULTIPART_SPLIT=10000000

data/lib/cnvrg/dataset.rb

@@ -4,7 +4,7 @@ module Cnvrg
     attr_reader :slug, :owner, :title, :local_path, :working_dir
 
     RemoteURL ||= "https://cnvrg.io"
-    IDXParallelThreads ||= 15
+    IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
     IDXParallelProcesses ||= Parallel.processor_count
 
     def initialize(project_home = '', dataset_url: '')

data/lib/cnvrg/helpers.rb

@@ -2,9 +2,15 @@ module Cnvrg
   module Helpers
 
     extend self
-    ParallelThreads ||= 20
     def parallel_threads()
-      return ParallelThreads
+      threads = ENV["CNVRG_PARALLEL_THREADS"].to_i
+
+      
+      ### if empty, default will be 15 threads
+      threads = threads > 0 ? threads : 15
+
+      ### set max threads to be 100k
+      [threads, 100000].min
     end
     def checkmark
       checkmark = "\u2713"

data/lib/cnvrg/project.rb

@@ -4,7 +4,7 @@ module Cnvrg
     attr_reader :slug, :owner, :title, :local_path, :working_dir, :is_git, :is_branch, :machines
 
     RemoteURL ||= "https://cnvrg.io"
-    IDXParallelThreads ||= 15
+    IDXParallelThreads ||= Cnvrg::Helpers.parallel_threads
 
 
     def initialize(project_home)
@@ -632,5 +632,52 @@ module Cnvrg
     end
 
 
+    def spot_will_terminate
+      restart = false
+      config = self.get_config
+      # logic that will prevent multiple restart calls
+      if config[:spot_taken]
+        return false
+      end
+      begin
+        url = URI.parse('http://169.254.169.254/latest/meta-data/spot/termination-time')
+        req = Net::HTTP::Get.new(url.to_s)
+        res = Net::HTTP.start(url.host, url.port) {|http|
+          http.request(req)
+        }
+        unless res.body.include? "404"
+          restart = true
+        end
+        if res.body.include? "Empty reply from server"
+          restart = false
+        end
+      rescue
+        restart = false
+      end
+
+      if restart
+        config[:spot_taken] = true
+        self.set_config(config)
+      end
+
+      return restart
+    end
+
+    def send_restart_request(job_id: nil, job_type: nil, ma_id: nil)
+      Cnvrg::API.request("#{base_resource}/spot_restart", 'POST', {job_type: job_type, job_id: job_id, machine_activity: ma_id})
+    end
+
+    def get_machine_activity
+      begin
+        machine_activity = File.open("#{@local_path}/.cnvrg/machine_activity", "rb").read
+        machine_activity = machine_activity.to_s.strip
+        ma_id = machine_activity.to_i
+        return ma_id
+      rescue
+        return nil
+      end
+    end
+
+
   end
 end

data/lib/cnvrg/version.rb

@@ -1,4 +1,4 @@
 module Cnvrg
-  VERSION = '1.2.6'
+  VERSION = '1.2.7'
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cnvrg
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.2.7
 platform: ruby
 authors:
 - Yochay Ettun
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-20 00:00:00.000000000 Z
+date: 2019-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler