hive-runner 2.1.21 → 2.1.22

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b2497d9a253abc7f2b461aead5c9711003afdde
-  data.tar.gz: a86a5c4c3a66365dbef517639dd55e3685c99016
+  metadata.gz: 216a3488b467928dd26057df38e750217a385cee
+  data.tar.gz: 53e45b27b62accdeac7c55ec6adcc4ee82cc577b
 SHA512:
-  metadata.gz: 6ef83adef1c0bdf1301d3da224d5764805ea31acccc465e01acf85de0c637557c3884cffc143e3fd1569d3cc97d58f868a3919d5cc0b1a0aef1fde507fd05c58
-  data.tar.gz: c0d4ff7e2fa5f41606b533c4bce1b34a84060ee5b75d813131a75f00b1562603d7b2f90ef85dfee7e6a4cd3fef4da65efd8e32f474c83034ca3148c7709012b1
+  metadata.gz: 0bf074173c966e0e639cc9826e787581b8a96afa3674589ed271e57c431b61a75fc22bbd74097aec091c98a8f60ec6be6868c521406422fb8b2d9f91368f0cf7
+  data.tar.gz: dd5a30d68ce66bf0e423ccc164fb289f663300f1379ed5f406b5af7dcea76deb1083708a6d061ca57759904c6939186f1f65b622a2c13f3d8cba76c618844e1f

data/lib/hive/execution_script.rb

@@ -95,8 +95,13 @@ module Hive
             running = false
           end
         rescue Timeout::Error
-          Process.kill(-9, @pgid) if ! ( @keep_running.nil? || @keep_running.call )
-          # Do something. Eg, upload log files.
+          @log.debug("Sub-process keep_running check")
+          if ! ( @keep_running.nil? || @keep_running.call )
+             Process.kill(-9, @pgid)
+             raise TimeoutError.new("Timed out after #{Hive.config.timings.job_timeout} seconds")
+          end
+          
+          # TODO Upload in-progress script logs
         end
       end
 

data/lib/hive/worker.rb

@@ -31,6 +31,7 @@ module Hive
       @device_id = @options['id']
       @hive_id = @options['hive_id']
       @default_component ||= self.class.to_s
+      @current_job_start_time = nil
       @hive_mind ||= mind_meld_klass.new(
         url: Chamber.env.network.hive_mind? ? Chamber.env.network.hive_mind : nil,
         pem: Chamber.env.network.cert ? Chamber.env.network.cert : nil,
@@ -68,7 +69,7 @@ module Hive
       end
 
       @log.info('Starting worker')
-      while keep_running?
+      while keep_worker_running?
         begin
           @log.clear
           update_queues
@@ -91,6 +92,7 @@ module Hive
       else
         @log.info('Job starting')
         begin
+          @current_job_start_time = Time.now
           execute_job
         rescue => e
           @log.info("Error running test: #{e.message}\n : #{e.backtrace.join("\n :")}")
@@ -164,7 +166,7 @@ module Hive
         @script = Hive::ExecutionScript.new(
           file_system: @file_system,
           log: @log,
-          keep_running: ->() { self.keep_running? }
+          keep_running: ->() { self.keep_script_running? }
         )
         @script.append_bash_cmd "mkdir -p #{@file_system.testbed_path}/#{@job.execution_directory}"
         @script.append_bash_cmd "cd #{@file_system.testbed_path}/#{@job.execution_directory}"
@@ -379,14 +381,48 @@ module Hive
       CodeCache.repo(repository).checkout(:head, checkout_directory, branch) or raise "Unable to checkout repository #{repository}"
     end
 
-    # Determine whether to keep the worker running
-    # This just checks the presense of the parent process
-    def keep_running?
+    # Keep the worker process running
+    def keep_worker_running?
+      @log.debug("Keep Worker Running check ")
+      if parent_process_dead?
+        @log.info("Think parent process is dead")
+        false
+      else
+        true
+      end
+    end
+    
+    # Keep the execution script running
+    def keep_script_running?
+      @log.debug("Keep Running check ")
+      if exceeded_time_limit? || parent_process_dead?
+        return false
+      else
+        return true
+      end
+    end
+
+    def exceeded_time_limit?
+      if @job && !@job.nil?
+        if max_time = @job.execution_variables.job_timeout rescue nil
+          elapsed = (Time.now - @current_job_start_time).to_i
+          @log.debug("Elapsed = #{elapsed} seconds, Max = #{max_time} minutes")
+          if elapsed > max_time.to_i * 60          
+            @log.warn("Job has exceeded max time of #{max_time} minutes")
+            return true 
+          end
+        end
+      end
+      false
+    end
+
+    def parent_process_dead?
       begin
         Process.getpgid(@parent_pid)
-        true
-      rescue
         false
+      rescue
+        @log.warn("Parent process appears to have terminated")
+        true
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hive-runner
 version: !ruby/object:Gem::Version
-  version: 2.1.21
+  version: 2.1.22
 platform: ruby
 authors:
 - Joe Haig
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-25 00:00:00.000000000 Z
+date: 2017-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: chamber