rsched 0.2.0 → 0.3.0

data/ChangeLog

@@ -0,0 +1,18 @@
+
+== 2011-07-19 version 0.3.0
+
+* Changed not to release a lock when a task is failed
+* Added -E, --release-on-fail option
+* Added respectable timeout implementation
+
+
+== 2011-07-11 version 0.2.0
+
+* Splits single bin file into multiple files
+* Added test codes
+
+
+== 2011-07-10 version 0.1.0
+
+* Initial release
+

data/lib/rsched/command/rsched.rb

@@ -9,13 +9,15 @@ confout = nil
 schedule = []
 
 defaults = {
-  :timeout => 600,
+  :timeout => 30,
   :resume => 3600,
   :delete => 2592000,
   :delay => 0,
   :interval => 10,
   :type => 'mysql',
   :name => "#{Process.pid}.#{`hostname`.strip}",
+  :kill_retry => 60,
+  :release_on_fail => false,
 }
 
 conf = { }
@@ -36,7 +38,7 @@ op.on('-a', '--add EXPR', 'Add an execution schedule') {|s|
   schedule << s
 }
 
-op.on('-t', '--timeout SEC', 'Retry timeout (default: 600)', Integer) {|i|
+op.on('-t', '--timeout SEC', 'Retry timeout (default: 30)', Integer) {|i|
   conf[:timeout] = i
 }
 
@@ -64,14 +66,26 @@ op.on('-F', '--from YYYY-mm-dd_OR_now', 'Time to start scheduling') {|s|
   end
 }
 
-#op.on('-x', '--kill-timeout SEC', 'Threashold time before killing process (default: timeout * 5)', Integer) {|i|
-#  conf[:kill_timeout] = i
-#}
+op.on('-e', '--extend-timeout SEC', 'Threashold time before extending visibility timeout (default: timeout * 3/4)', Integer) {|i|
+  conf[:extend_timeout] = i
+}
+
+op.on('-x', '--kill-timeout SEC', 'Threashold time before killing process (default: timeout * 10)', Integer) {|i|
+  conf[:kill_timeout] = i
+}
+
+op.on('-X', '--kill-retry SEC', 'Threashold time before retrying killing process (default: 60)', Integer) {|i|
+  conf[:kill_retry] = i
+}
 
 op.on('-i', '--interval SEC', 'Scheduling interval (default: 10)', Integer) {|i|
   conf[:interval] = i
 }
 
+op.on('-E', '--release-on-fail', 'Releases lock if task failed so that other node can retry immediately', TrueClass) {|b|
+  conf[:release_on_fail] = b
+}
+
 op.on('-T', '--type TYPE', 'Lock database type (default: mysql)') {|s|
   conf[:type] = s
 }
@@ -177,6 +191,14 @@ begin
     raise "Unknown lock server type '#{conf[:type]}'"
   end
 
+  unless conf[:extend_timeout]
+    conf[:extend_timeout] = conf[:timeout] / 4 * 3
+  end
+
+  unless conf[:kill_timeout]
+    conf[:kill_timeout] = conf[:timeout] * 10
+  end
+
 rescue
   usage $!.to_s
 end
@@ -248,9 +270,16 @@ end
 if type == :run
   load File.expand_path(conf[:run])
   run_proc = method(:run)
+  if defined? terminate
+    kill_proc = method(:terminate)
+  else
+    kill_proc = Proc.new { }
+  end
 else
   run_proc = RSched::ExecRunner.new(conf[:exec])
+  kill_proc = run_proc.method(:terminate)
 end
 
-worker.run(run_proc)
+worker.init_proc(run_proc, kill_proc)
+worker.run
 

data/lib/rsched/dblock.rb

@@ -4,9 +4,8 @@ module RSched
 
 class DBLock < Lock
   def initialize(hostname, timeout, uri, user, pass)
+    super(hostname, timeout)
     require 'dbi'
-    @hostname = hostname
-    @timeout = timeout
     @db = DBI.connect(uri, user, pass)
     init_db
   end
@@ -47,6 +46,13 @@ class DBLock < Lock
     return n > 0
   end
 
+  def extend_timeout(token, timeout=Time.now.to_i+@timeout)
+    ident, time = *token
+    n = @db.do('UPDATE rsched SET timeout=? WHERE ident = ? AND time = ? AND host = ?;',
+           timeout, ident, time, @hostname)
+    return n > 0
+  end
+
   def delete_before(ident, time)
     @db.do('DELETE FROM rsched WHERE ident = ? AND time < ? AND finish IS NOT NULL;', ident, time)
   end
@@ -61,8 +67,8 @@ class DBLock < Lock
   end
 
   def try_update(ident, time, now)
-    n = @db.do('UPDATE rsched SET host=?, timeout=? WHERE ident = ? AND time = ? AND finish IS NULL AND (timeout < ? OR host = ?);',
-            @hostname, now+@timeout, ident, time, now, @hostname)
+    n = @db.do('UPDATE rsched SET host=?, timeout=? WHERE ident = ? AND time = ? AND finish IS NULL AND timeout < ?;',
+            @hostname, now+@timeout, ident, time, now)
     return n > 0
   end
 

data/lib/rsched/engine.rb

@@ -1,4 +1,5 @@
 require 'thread'
+require 'monitor'
 require 'time'
 require 'rsched/lock'
 
@@ -50,10 +51,16 @@ class Engine
     @delay = conf[:delay]
     @interval = conf[:interval]
     @delete = conf[:delete]
+    @extend_timeout = conf[:extend_timeout]
+    @kill_timeout = conf[:kill_timeout]
+    @kill_retry = conf[:kill_retry]
     @sched_start = conf[:from] || 0
+    @release_on_fail = conf[:release_on_fail]
     @finished = false
     @ss = {}
 
+    @extender = TimerThread.new(@lock, @extend_timeout, @kill_timeout, @kill_retry)
+
     @mutex = Mutex.new
     @cond = ConditionVariable.new
   end
@@ -64,7 +71,13 @@ class Engine
     @ss[ident] = Sched.new(cron, action, @sched_start, now-@resume, now-@delay)
   end
 
-  def run(run_proc)
+  def init_proc(run_proc, kill_proc)
+    @run_proc = run_proc
+    @extender.init_proc(kill_proc)
+  end
+
+  def run
+    @extender.start
     until @finished
       one = false
 
@@ -75,8 +88,8 @@ class Engine
         s.queue.delete_if {|time|
           next if @finished
 
-          x = @lock.acquire(ident, time)
-          case x
+          token = @lock.acquire(ident, time)
+          case token
           when nil
             # already finished
             true
@@ -87,16 +100,7 @@ class Engine
 
           else
             one = true
-            if process(ident, time, s.action, run_proc)
-              # success
-              @lock.finish(x)
-              try_delete(ident)
-              true
-            else
-              # fail
-              @lock.release(x)
-              false
-            end
+            process(token, ident, time, s.action)
           end
         }
 
@@ -114,6 +118,7 @@ class Engine
 
   def shutdown
     @finished = true
+    @extender.shutdown
     @mutex.synchronize {
       @cond.broadcast
     }
@@ -138,22 +143,120 @@ class Engine
     end
   end
 
-  def process(ident, time, action, run_proc)
+  def process(token, ident, time, action)
+    puts "started token=#{token.inspect} time=#{time}"
+
+    @extender.set_token(token)
+
+    success = false
     begin
-      run_proc.call(ident, time, action)
-      return true
+      @run_proc.call(ident, time, action)
+      puts "finished token=#{token.inspect}"
+      success = true
     rescue
-      puts "failed ident=#{ident} time=#{time}: #{$!}"
+      puts "failed token=#{token.inspect} time=#{time}: #{$!}"
       $!.backtrace.each {|bt|
         puts "  #{bt}"
       }
-      return false
+    end
+
+    @extender.reset_token
+
+    if success
+      @lock.finish(token)
+      cleanup_old_entries(ident)
+      true
+    else
+      if @release_on_fail
+        @lock.release(token)
+      end
+      false
     end
   end
 
-  def try_delete(ident)
+  def cleanup_old_entries(ident)
     @lock.delete_before(ident, Time.now.to_i-@delete)
   end
+
+  class TimerThread
+    include MonitorMixin
+
+    def initialize(lock, extend_timeout, kill_timeout, kill_retry)
+      super()
+      @lock = lock
+      @extend_timeout = extend_timeout
+      @kill_timeout = kill_timeout
+      @kill_retry = kill_retry
+      @kill_time = nil
+      @kill_proc = nil
+      @extend_time = nil
+      @token = nil
+      @finished = false
+    end
+
+    def init_proc(kill_proc)
+      @kill_proc = kill_proc
+    end
+
+    def start
+      @thread = Thread.new(&method(:run))
+    end
+
+    def join
+      @thread.join
+    end
+
+    def set_token(token)
+      synchronize do
+        now = Time.now.to_i
+        @extend_time = now + @extend_timeout
+        @kill_time = now + @kill_timeout
+        @token = token
+      end
+    end
+
+    def reset_token
+      synchronize do
+        @token = nil
+      end
+    end
+
+    def shutdown
+      @finished = true
+    end
+
+    private
+    def run
+      until @finished
+        sleep 1
+        synchronize do
+          if @token
+            now = Time.now.to_i
+            try_kill(now, @token)
+            try_extend(now, @token)
+          end
+        end
+      end
+    end
+
+    def try_extend(now, token)
+      if now > @extend_time
+        puts "extending token=#{token.inspect}"
+        @lock.extend_timeout(token, now)
+        @extend_time = now + @extend_timeout
+      end
+    end
+
+    def try_kill(now, token)
+      if now > @kill_time
+        if @kill_proc
+          puts "killing #{token.inspect}..."
+          @kill_proc.call rescue nil
+        end
+        @kill_time = now + @kill_retry
+      end
+    end
+  end
 end
 
 
@@ -161,11 +264,14 @@ class ExecRunner
   def initialize(cmd)
     @cmd = cmd + ' ' + ARGV.map {|a| Shellwords.escape(a) }.join(' ')
     @iobuf = ''
+    @pid = nil
+    @next_kill = :TERM
   end
 
   def call(ident, time, action)
     message = [ident, time, action].join("\t")
     IO.popen(@cmd, "r+") {|io|
+      @pid = io.pid
       io.write(message) rescue nil
       io.close_write
       begin
@@ -180,6 +286,11 @@ class ExecRunner
       raise "Command failed"
     end
   end
+
+  def terminate
+    Process.kill(@next_kill, @pid)
+    @next_kill = :KILL
+  end
 end
 
 

data/lib/rsched/lock.rb

@@ -4,16 +4,23 @@ module RSched
 
 class Lock
   def initialize(hostname, timeout)
+    @hostname = hostname
+    @timeout = timeout
   end
 
+  attr_reader :hostname, :timeout
+
   # acquired=token, locked=false, finished=nil
-  def acquire(ident, time)
+  def acquire(ident, time, now=Time.now.to_i)
+  end
+
+  def release(token, next_timeout=Time.now.to_i)
   end
 
-  def release(token)
+  def finish(token, now=Time.now.to_i)
   end
 
-  def finish(token)
+  def extend_timeout(token, timeout=Time.now.to_i+@timeout)
   end
 
   def delete_before(ident, time)

data/lib/rsched/version.rb

@@ -1,5 +1,5 @@
 module RSched
 
-VERSION = '0.2.0'
+VERSION = '0.3.0'
 
 end

data/test/dblock_test.rb

@@ -34,10 +34,11 @@ class DBLockTest < Test::Unit::TestCase
     token = db2.acquire('ident1', time, now)
     assert_equal false, token
 
-    # same host can relock
-    token = db1.acquire('ident1', time, now)
-    assert_not_equal nil, token
-    assert_not_equal false, token
+    # v0.4.0: same host can't relock
+    ## same host can relock
+    #token = db1.acquire('ident1', time, now)
+    #assert_not_equal nil, token
+    #assert_not_equal false, token
 
     # different identifier
     token = db2.acquire('ident2', time, now)
@@ -135,17 +136,19 @@ class DBLockTest < Test::Unit::TestCase
     assert_not_equal nil, token
     assert_not_equal false, token
 
-    # same host can relock
-    # + extend timeout
-    token = db1.acquire('ident1', time, now+TIMEOUT)
-    assert_not_equal nil, token
-    assert_not_equal false, token
+    # different host can't extend (even if same token)
+    ok = db2.extend_timeout(token, now+TIMEOUT*2)
+    assert_equal false, ok
+
+    # same host can extend timeout
+    ok = db1.extend_timeout(token, now+TIMEOUT*2)
+    assert_equal true, ok
 
-    # timeout is extended
+    # timeout is extended; different host can't lock
     token_ = db2.acquire('ident1', time, now+TIMEOUT+1)
     assert_equal false, token_
 
-    # extended timeout is expired
+    # extended timeout is expired; different host can lock
     token = db2.acquire('ident1', time, now+TIMEOUT*2+1)
     assert_not_equal nil, token
     assert_not_equal false, token

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: rsched
 version: !ruby/object:Gem::Version 
-  hash: 23
+  hash: 19
   prerelease: false
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Sadayuki Furuhashi
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-12 00:00:00 +09:00
+date: 2011-07-18 00:00:00 +09:00
 default_executable: rsched
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -57,6 +57,7 @@ executables:
 extensions: []
 
 extra_rdoc_files: 
+- ChangeLog
 - README.rdoc
 files: 
 - bin/rsched
@@ -69,6 +70,7 @@ files:
 - test/exec_test.rb
 - test/sched_test.rb
 - test/test_helper.rb
+- ChangeLog
 - README.rdoc
 - test/cat.sh
 - test/fail.sh