rescheduler 0.3.0 → 0.4.0

data/lib/rescheduler.rb

@@ -1,4 +1,5 @@
-require 'json'
+require 'time' # Needed for Time.parse
+require 'multi_json'
 require 'redis'
 
 module Rescheduler
@@ -8,6 +9,10 @@ module Rescheduler
   attr_accessor :config
   self.config = {}
 
+  # Debugging only (since initializers won't reload)
+  self.config[:prefix] = REDIS_PREFIX # This ensures cross DB deployment persistence
+  self.config[:redis] = RedisHelper.redis
+
   #==========================
   # Management routines
   def prefix
@@ -22,7 +27,8 @@ module Rescheduler
     redis.del(keys)
   end
 
-  # Return a hash of statistics
+  # Return a hash of statistics, in this format
+  # 
   def stats
     loop do 
       redis.watch(rk_args) do
@@ -76,19 +82,36 @@ module Rescheduler
     end
   end
 
+  # NOTE: Use this with care. Some lost jobs can be moved to immediate queue instead of deleted
+  # Pass '*' to delete everything.
+  def purge_bad_jobs(queue = '*')
+    pending, running, deferred = redis.multi do 
+      redis.hkeys(rk_args)
+      redis.hkeys(rk_running)
+      redis.zrange(rk_deferred, 0, -1)
+    end
+
+    bad = pending - running - deferred
+    bad.each do |qnid| 
+      next if queue != '*' && !qnid.start_with?(queue + ':')
+      idelete(qnid)
+    end
+  end
+
   #==========================
   # Task producer routines
   # Add an immediate task to the queue
   def enqueue(options=nil)
     options ||= {}
-    now = Time.now
+    now = Time.now.to_i
 
     # Error check 
     validate_queue_name(options[:queue]) if options.include?(:queue)
+    validate_recurrance(options)
 
     # Convert due_in to due_at
     if options.include?(:due_in)
-      raise ArgumentError, ':due_it and :due_at can not be both specified' if options.include?(:due_at)
+      raise ArgumentError, ':due_in and :due_at can not be both specified' if options.include?(:due_at)
       options[:due_at] = now + options[:due_in]
     end
 
@@ -98,7 +121,9 @@ module Rescheduler
       options[:id] = redis.incr(rk_counter)
     end
 
-    ts = options[:due_at].to_i || 0
+    ts = options[:due_at].to_i
+    ts = now if ts == 0 # 0 means immediate
+    options[:due_at] = ts # Convert :due_at to integer timestamp to be reused in recurrance
     qnid = get_qnid(options)
 
     # Encode and save args
@@ -109,18 +134,18 @@ module Rescheduler
       end
 
       # Save options
-      redis.hset(rk_args, qnid, options.to_json)
+      redis.hset(rk_args, qnid, MultiJson.dump(options))
 
       # Determine the due time
-      if ts > now.to_i # Future job
+      if ts > now # Future job
         redis.zadd(rk_deferred, ts, qnid)
       else
         redis.lpush(rk_queue(options[:queue]), qnid)
       end
     end
 
-    # Now decide if we need to wait up the workers (outside of the transaction)
-    if (ts > now.to_i)
+    # Now decide if we need to wake up the workers (outside of the transaction)
+    if (ts > now)
       dt = redis.zrange(rk_deferred, 0, 0)[0]
       # Wake up workers if our job is the first one in deferred queue, so they can reset timeout
       if dt && dt == qnid
@@ -142,14 +167,58 @@ module Rescheduler
 
   def delete(options)
     qnid = get_qnid(options)
-    
-    redis.multi do
-      redis.hdel(rk_args, qnid)
-      redis.zrem(rk_deferred, qnid)
-      redis.lrem(rk_queue(options[:queue]), 0, qnid)
+    idelete(qnid)
+  end
+
+  # Make a job immediate if it is not already. Erase the wait
+  def make_immediate(options)
+    dtn = rk_deferred # Make a copy in case prefix changes
+    qnid = get_qnid(options)
+    ntry = 0
+    loop do
+      redis.watch(dtn) do 
+        if redis.zcard(dtn, qnid) == 0
+          redis.unwatch(dtn)
+          return # Not a deferred job
+        else
+          redis.multi
+          redis.zrem(dtn, qnid)
+          q = qnid_to_queue(qnid)
+          redis.lpush(rk_queue(q), qnid)
+          if !redis.exec
+            # Contention happens, retrying
+            log_debug("make_immediate contention for #{qnid}")
+            Kernel.sleep (rand(ntry * 1000) / 1000.0) if ntry > 0
+          else
+            return # Done
+          end
+        end
+      end
+      ntry += 1
     end
   end
 
+  #=================
+  # Serialization (in case it is needed to transfer all Rescheduler across to another redis instance)
+
+  # Atomically save the state to file and stop all workers (state in redis is not destroyed)
+  # This function can take a while as it will wait for running jobs to finish first.
+  def serialize_and_stop(filename)
+    # TODO
+  end
+
+  # Load state from a file. Will merge into existing jobs if there are any (make sure it is done only once)
+  # This can be done before any worker starts, or after. 
+  # Workers still need to be manually started
+  def deserialize(filename)
+    # TODO
+  end
+
+  # Clear redis states and delete all jobs (useful before deserialize)
+  def erase_all
+    # TODO
+  end
+
   #=================
   # Job definition
   # Task consumer routines
@@ -159,6 +228,16 @@ module Rescheduler
     return nil
   end
 
+  #=================
+  # Error handling
+  def on_error(tube=nil, &block)
+    if tube != nil
+      @error_handlers ||= {}
+      @error_handlers[tube] = block
+    else
+      @global_error_handler = block;
+    end
+  end
   #=================
   # Runner/Maintenance routines
   def start(*tubes)
@@ -217,9 +296,19 @@ module Rescheduler
 
   private 
 
+  # Internal routines operating out of qnid
+  def idelete(qnid)
+    queue = qnid.split(':').first
+    redis.multi do
+      redis.hdel(rk_args, qnid)
+      redis.zrem(rk_deferred, qnid)
+      redis.lrem(rk_queue(queue), 0, qnid)
+    end
+  end
+
   # Runner routines
   def run_job(qnid)
-    # First load job parameters for running
+    # 1. load job parameters for running
     optstr = nil
     begin
       res = nil
@@ -237,36 +326,64 @@ module Rescheduler
         end
         if !res
           # Contention, try read again
-          log_debug("Job read contention")
+          log_debug("Job read contention: (#{qnid})")
         end
       end
     end until res
 
     # Parse and run
-    opt = JSON.parse(optstr)
-    #opt.symbolize_keys # Be mindful of non-rails people, explicitly here
-    sopt = {}
-    opt.each { |key, val| sopt[key.to_sym] = val }
+    sopt = MultiJson.load(optstr, :symbolize_keys => true)
+
+    # Handle parameters
+    if (sopt.include?(:recur_every))
+      newopt = sopt.dup
+      newopt[:due_at] = (sopt[:due_at] || Time.now).to_i + sopt[:recur_every].to_i
+      newopt.delete(:due_in) # In case the first job was specified by :due_in
+      enqueue(newopt)
+    end
+
+    if (sopt.include?(:recur_daily))
+      newopt = sopt.dup      
+      newopt[:due_at] = time_from_recur_daily(sopt[:recur_daily])
+      newopt.delete(:due_in) # In case the first job was specified by :due_in
+      enqueue(newopt)
+    end
 
     # 2. Find runner and invoke it
     begin
       log_debug(">>---- Starting #{qnid}")
       runner = @runners[qnid_to_queue(qnid)]
-      runner.call(sopt)
-      log_debug("----<< Finished #{qnid}")
-    rescue 
-      log_debug("----<< Failed #{qnid}: #{$!}")
+      if runner.is_a?(Proc)
+        runner.call(sopt)
+        log_debug("----<< Finished #{qnid}")
+      else
+        log_debug("----<< Failed #{qnid}: Unknown queue name, handler not defined")
+      end
+    rescue Exception => e
+      log_debug("----<< Failed #{qnid}: -------------\n #{$!}")
+      log_debug(e.backtrace[0..4].join("\n"))
+      handle_error(e, qnid, sopt)
+      log_debug("------------------------------------\n")
     end
 
     # 3. Remove job from running list (Done)
     redis.hdel(rk_running, qnid)
   end
 
+  def handle_error(e, qnid, sopt)
+    error_handler = @error_handlers && @error_handlers[qnid]
+    if error_handler
+      error_handler.call(e, sopt) 
+    elsif @global_error_handler
+      @global_error_handler.call(e, sopt) 
+    end
+  end
+
   # Helper routines
 
   # Find all the "due" deferred jobs and move them into respective queues
   def service_deferred_jobs
-    dtn = rk_deferred
+    dtn = rk_deferred # Make a copy in case prefix changes
     ntry = 0
     loop do
       curtime = Time.now.to_i
@@ -333,16 +450,49 @@ module Rescheduler
   end
 
   def redis
-    @redis ||= Redis.new(@config[:redis_connection] || {})
+    @redis ||= @config[:redis] || Redis.new(@config[:redis_connection] || {})
   end
 
   def validate_queue_name(queue)
     raise ArgumentError, 'Queue name can not contain special characters' if queue.include?(':')
   end
 
+  def parse_seconds_of_day(recur_daily)
+    return recur_daily if recur_daily.is_a?(Fixnum)
+    time = Time.parse(recur_daily)
+    return time.to_i - Time.new(time.year, time.month, time.day).to_i
+  end
+
+  # Find the next recur time
+  def time_from_recur_daily(recur_daily, now = Time.now)
+    recur = parse_seconds_of_day(recur_daily)
+    t = Time.new(now.year, now.month, now.day).to_i + recur
+    t += 86400 if t < now.to_i
+    return Time.at(t)
+  end
+
+  def validate_recurrance(options)
+    rcnt = 0
+    if (options.include?(:recur_every))
+      rcnt += 1
+      raise 'Expect integer for :recur_every parameter' unless options[:recur_every].is_a?(Fixnum)
+    end
+
+    if (options.include?(:recur_daily))
+      rcnt += 1
+      time = time_from_recur_daily(options[:recur_daily]) # Try parse and make sure we can
+      unless options.include?(:due_at) || options.include?(:due_in)
+        options[:due_at] = time # Setup the first run
+      end
+    end
+    raise 'Can only specify one recurrance parameter' if rcnt > 1
+  end
+
   # Logging facility
   def log_debug(msg)
-    print("#{Time.now.to_i} #{msg}\n")
+    return if @config[:silent]
+    print("#{Time.now.iso8601} #{msg}\n")
+    STDOUT.flush
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rescheduler
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-01 00:00:00.000000000 Z
+date: 2014-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
@@ -28,7 +28,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: multi_json
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
@@ -72,7 +72,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.25
 signing_key: 
 specification_version: 3
 summary: A job queue for immediate and delayed jobs using Redis