task-orchestrator 0.0.9 → 0.0.10

data/examples/flock

@@ -0,0 +1,12 @@
+orchestrator:
+  statedir: states
+  flock:
+    description: flocking statefile test
+    save: true
+    steps:
+      - type: sequential
+        scripts:
+           - command: sleep 10
+           - command: sleep 10
+           - command: sleep 10
+           - command: sleep 10

data/examples/timeouts

@@ -8,10 +8,10 @@ orchestrator:
 #        on_failure: die
         on_failure: ignore
         scripts:
-           - command: sleep 5
-             timeout: 2
-           - command: sleep 1
-             timeout: 2
+           - command: sleep 50
+             timeout: 10
+           - command: sleep 10
+             timeout: 5
   timeouts2:
     description: timeouts test 2
     save: true

data/lib/orchestrator/cli.rb

@@ -32,7 +32,10 @@ module Orchestrator
       end
 
       options.reset = false
-      parser.on( '--reset', 'Do not use state file if it exists' ) { |reset| options.reset = true }
+      parser.on( '--reset', 'Do not use state file if it exists' ) { options.reset = true }
+
+      options.kill = false
+      parser.on( '--kill', 'Kill running task based on statefile pid then lock can not be acquired' ) { options.kill = true }
 
       parser.on( '--args ARGS,', Array, 'extra args for interpolation as arg1=val1,arg2=val2,...]' ) do |extra_args|
         extra_args.each do |extra_arg|

data/lib/orchestrator/task.rb

@@ -14,6 +14,8 @@ module Orchestrator
       Thread.abort_on_exception = true
       @log = ''
 
+      @children = {}
+
       @options = options
 
       @defaults = {}
@@ -32,7 +34,52 @@ module Orchestrator
           FileUtils.mkdir_p(@settings['orchestrator']['statedir'])
         end
       end
-      @state = (@options.statefile && File.exist?(@options.statefile) && !@options.reset) ? YAML.load_file(@options.statefile) : @settings['orchestrator'][@options.name]
+
+      if @options.statefile
+        if File.exist?(@options.statefile)
+          File.open(@options.statefile, "r") do |file|
+            @got_lock = file.flock( File::LOCK_NB | File::LOCK_EX )
+          end
+          unless @got_lock
+            if @options.kill
+              @state = YAML.load_file(@options.statefile)
+              begin
+                Process.kill("TERM", @state['pid']) if @state.has_key?('pid')
+              rescue Errno::ESRCH
+              end
+              sleep 0.1
+              File.open(@options.statefile, "r") do |file|
+                @got_lock = file.flock( File::LOCK_NB | File::LOCK_EX )
+              end
+            end
+            invalid("The state file #{@options.statefile} is already locked by other process") unless @got_lock
+          end
+        end
+
+        @state = (File.exist?(@options.statefile) && !@options.reset) ? YAML.load_file(@options.statefile) : @settings['orchestrator'][@options.name]
+
+        @state['pid'] = Process.pid
+
+        @statefile_handle = File.open(@options.statefile, "w")
+        @statefile_handle.flock( File::LOCK_NB | File::LOCK_EX )
+        @statefile_handle.sync = true
+        ### @statefile_handle.close_on_exec = true
+        Signal.trap("TERM") do
+          @children.each do |pid, command|
+            begin
+              Process.kill("TERM", pid)
+              puts "KILLED #{pid}: #{command}\n" if @options.verbose
+              @log += "KILLED #{pid}: #{command}\n"
+            rescue Errno::ESRCH
+            end
+          end
+          puts "KILLED ORCHESTRATOR\n" if @options.verbose
+          @log += "KILLED ORCHESTRATOR\n"
+          fail
+        end
+      else
+        @state = @settings['orchestrator'][@options.name]
+      end
 
       @options.email = false unless @state.has_key?('email')
       @options.email_on_success = (@options.email and @state['email'].has_key?('on_success')) ? @state['email']['on_success'] : true
@@ -49,7 +96,14 @@ module Orchestrator
     def save_state
       if @options.statefile
         @mutex.synchronize do
-          File.open(@options.statefile, "w") {|f| YAML.dump(@state, f)}
+          unless File.exist?(@options.statefile)
+            @statefile_handle.close
+            @statefile_handle = File.open(@options.statefile, "w")
+            @statefile_handle.flock( File::LOCK_NB | File::LOCK_EX )
+            @statefile_handle.sync = true
+          end
+          @statefile_handle.rewind
+          YAML.dump(@state, @statefile_handle)
         end
       end
     end
@@ -185,9 +239,12 @@ module Orchestrator
       #  start = Time.now
       
       status = 'STARTED'
+      child = nil
       begin
         Timeout::timeout(timeout) do
           status = POpen4::popen4(command) do |stdout, stderr, stdin, pid|
+            @children[pid] = command
+            child = pid
             result = stdout.read.strip
             error = stderr.read.strip
           end
@@ -195,12 +252,21 @@ module Orchestrator
         end
       rescue Timeout::Error
         status = 'TIMEOUT'
+        if child
+          begin
+            Process.kill("TERM", child)
+            result += "\nKILLED #{child}: #{command}\n"
+          rescue Errno::ESRCH
+          end
+        end
       end
 
       #  runtime = Time.now - start
       #  runtime = runtime > 60 ? runtime/60 : runtime
 
       @mutex.synchronize do
+        @children.delete(child) if child
+
         output = <<-EOF
 
 Running: #{command} - #{status}

data/lib/orchestrator/version.rb

@@ -1,3 +1,3 @@
 module Orchestrator
-  VERSION ||= '0.0.9'
+  VERSION ||= '0.0.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: task-orchestrator
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.0.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-11 00:00:00.000000000 Z
+date: 2013-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pony
@@ -85,16 +85,17 @@ extra_rdoc_files: []
 files:
 - bin/orchestrator
 - lib/orchestrator.rb
-- lib/orchestrator/task.rb
 - lib/orchestrator/version.rb
 - lib/orchestrator/cli.rb
+- lib/orchestrator/task.rb
+- examples/timeouts
+- examples/interpolation
+- examples/dependencies
 - examples/sequential
 - examples/failure_handler
-- examples/multistep
 - examples/parallel
-- examples/interpolation
-- examples/timeouts
-- examples/dependencies
+- examples/flock
+- examples/multistep
 - task-orchestrator.gemspec
 - LICENSE
 - README.md