rescheduler 0.4.1 → 0.5.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5e0575537baa956f4e8628a7892d9bcc0c3c6a80
+  data.tar.gz: 1481a1e1b3f2f4e491787793466db8479235ff61
+SHA512:
+  metadata.gz: 2107fe553f4b8a9570b8623d5d46b46552d8ead112201c7d80dd62df309ad280b3fb9d3c3876995ac02674a035e02f6d5f2653e959b03b346fdf589e2600c26a
+  data.tar.gz: f897e041abbab34c119c70aad94c9333c4f4c94b52d72940b72483844d2be4f38eea31de2d38bc91401c6e2f7c9e3083db0b2b1b014926f1d383a7f5c715a844

data/bin/rescheduler_launch

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+=begin
+rescheduler_launch <options> [<worker_name> <jobs.rb>]
+Options:
+  --rails=<folder>    Use rails environment from the givne folder
+  --log=<file>        Specify logfile (in Rails mode default to <rails_foler>/log/<worker_name>.log)
+  --env=<env>         Set these ENV into the new process
+  --debug             Do not daemonize or redirect output, work with current STDOUT
+  --respawn_jobs=N    Respawn worker after N jobs
+  --respawn_time=N    Respawn every N seconds (NOTE: Only will respawn after finish a job)
+Internal purpose only:
+  --respawn           Set to indicate this is a respawn of the named worker and will take over the
+                      name without deleting stats
+=end
+
+# First parse options
+opts = {}
+while ARGV.size > 0 && ARGV[0][0..1] == '--'
+  opt,val = ARGV.shift[2..-1].split('=',2)
+  val ||= true
+  opts[opt] = val
+end
+
+opts['worker_name'] = ARGV.shift
+opts['worker_file'] = ARGV.shift
+
+abort "rescheduler_launch <options> <worker_name> <jobs.rb>" unless opts['worker_file'] && File.exists?(opts['worker_file'])
+
+# Daemonize first before we load rails
+begin
+  Process.daemon unless opts['debug'] # Do not daemonize if debugging
+rescue NotImplementedError
+  # This happens in windows, it is OK
+end
+
+# Load rails environment if --rails is specified
+require File.join(opts['rails'], 'config/environment.rb') if opts['rails']
+
+# Load rescheduler
+require File.expand_path('../../lib/rescheduler', __FILE__)
+
+Rescheduler::Worker.register(opts)
+Rescheduler::Worker.redirect_logging(opts) unless opts['debug']
+
+# Running the job file
+begin
+  Kernel.load opts['worker_file']
+ensure
+  Rescheduler::Worker.unregister unless Rescheduler::Worker.respawn_if_requested
+end

data/lib/rescheduler/sync.rb

@@ -0,0 +1,44 @@
+module Rescheduler
+  module Sync
+    extend self
+
+    def lock(name, opts={})
+      raise "Requires a block to be supplied" unless block_given? # Maybe later offer naked lock/unlocks?
+      raise "Need a valid string name" unless name.is_a?(String)
+
+      res = do_lock(name, opts) # This would block
+      return nil unless res # Timeout or failed somehow
+
+      begin
+        yield
+      ensure
+        do_unlock(name)
+      end
+      return true # Lock was successful
+    end
+
+    # Forcefully unlock and delete existing locks
+    def clear!(name)
+      redis.del(rk_exists_name(name), rk_lock_name(name))
+    end
+
+    private def do_lock(name, opts)
+      # Make sure semaphore for a given name is only created once
+      if redis.getset(rk_exists_name(name), 1)
+        # Already created, block wait for the release (possibility for unlock)
+        return redis.brpop(rk_lock_name(name), timeout: (opts[:timeout] || 0))
+      else
+        # First time, get the lock automatically
+        return true
+      end
+    end
+
+    private def do_unlock(name)
+      redis.lpush(rk_lock_name(name), 1)
+    end
+
+    private def redis; Rescheduler.send(:redis); end
+    private def rk_exists_name(name); return "#{Rescheduler.prefix}Sync:#{name}"; end
+    private def rk_lock_name(name); return "#{Rescheduler.prefix}SyncQ:#{name}"; end
+  end
+end

data/lib/rescheduler/worker.rb

@@ -0,0 +1,293 @@
+require 'socket'
+
+module Rescheduler
+  module Worker
+    extend self
+    # Each worker is a process of it own, handled by this singleton module
+
+    NONCMD_OPTS = %w[pid machine worker_name worker_file]
+    CMD_OPTS = %w[rails log env respawn] # respanw is for internal only
+    attr_accessor :launch_options
+    def worker_name; @launch_options && @launch_options['worker_name']; end
+    def worker_index; @launch_options && @launch_options['worker_index'] || -1; end
+    # ====================================================================
+    # Controller
+    # ====================================================================
+    WORKERCMD_SUSPEND = 'suspend'
+    WORKERCMD_RESUME = 'resume'
+    WORKERCMD_STOP = 'stop'
+    WORKERCMD_RESTART = 'restart'
+    [WORKERCMD_SUSPEND, WORKERCMD_RESUME, WORKERCMD_STOP, WORKERCMD_RESTART].each do |cmd|
+      define_method(cmd) {|name| redis.lpush(rk_queue(name), cmd) }
+      define_method(cmd + '_all') {|pattern| cmd_to_pattern(pattern, cmd) }
+    end
+
+    def workers_from_pattern(pattern)
+      pattern = pattern.gsub('%', '*')
+      workers = redis.keys(rk_worker(pattern))
+      kl = rk_worker('').length
+      workers.map {|w| w[kl..-1]}
+    end
+
+    def cmd_to_pattern(pattern, cmd)
+      ws = []
+      workers_from_pattern(pattern).each do |wname|
+        redis.lpush(rk_queue(wname), cmd)
+        ws << wname
+      end
+      return ws
+    end
+
+    def kill_all(pattern, force=false)
+      workers_from_pattern(pattern).each do |wname|
+        kill(wname, force)
+      end
+    end
+
+    # Do not run this unless everything else fails, it kills unconditionally
+    def kill(worker_name, force=false)
+      key = rk_worker(worker_name)
+      opts = redis.hgetall(key)
+      if force || opts['machine'] == Socket.gethostname
+        stop(worker_name)
+        sleep 1 # Give the process a second to quick peacefully
+        redis.del(key)
+        # Actually kill the process
+        begin
+          pid = opts['pid']
+          # "HUP" does not get recognized in windows
+          Process.kill(9, pid.to_i) if pid && opts['machine'] == Socket.gethostname
+        rescue Errno::ESRCH, Errno::EPERM # No such process, Not permitted, these will be ginored
+        end
+      end
+    end
+
+    def clean_dead_workers
+      hostname = Socket.gethostname # Only can cleanup PIDs on own machine
+      workers = redis.keys(rk_worker('*'))
+      workers.each do |w|
+        opts = redis.hgetall(w)
+        next unless opts['machine'] == hostname
+        next if pid_exists?(opts['pid'])
+        print "Cleaning dead worker: #{w.split(':',3).last}\n"
+        redis.del(w) # Remove the record (Main part of the cleanup)
+      end
+      nil
+    end
+
+    def restart_self
+      Rescheduler.log_debug "[#{worker_name}] Restarting"
+      @respawn = true
+      Rescheduler.end_job_loop
+      @in_suspend = false
+    end
+
+    def handle_command(command)
+      case command
+      when WORKERCMD_SUSPEND
+        Rescheduler.log_debug "[#{worker_name}] Suspending"
+        @in_suspend = true
+        suspend_loop
+      when WORKERCMD_RESUME
+        if @in_suspend
+          Rescheduler.log_debug "[#{worker_name}] Resuming"
+          @in_suspend = false # This will resume the normal job loop
+        else
+          Rescheduler.log_debug "[#{worker_name}] Resume command received when not suspended"
+        end
+      when WORKERCMD_RESTART
+        restart_self
+      when WORKERCMD_STOP
+        Rescheduler.log_debug "[#{worker_name}] Stopping"
+        Rescheduler.end_job_loop
+        @in_suspend = false
+      else
+        Rescheduler.log_debug "[#{worker_name}] Unknown command: #{command}"
+      end
+    end
+
+    def suspend_loop
+      while @in_suspend do
+        result = redis.brpop(rk_queue)
+        if result
+          command = result[1]
+          handle_command(command)
+        end
+      end
+    end
+
+    # ====================================================================
+    # Runner
+    # ====================================================================
+
+    DEVNULL = '/dev/null'
+    def redirect_logging(opts)
+      return if windows_env? # Do not redirect in windows environment, let it run with cmd console
+
+      logfile = opts['log']
+      logfile ||= File.join(opts['rails'], "log/#{worker_name}.log") if opts['rails']
+      logfile ||= DEVNULL
+
+      # Redirect io
+      unless logfile == DEVNULL
+        # Code inspired by Daemonize::redirect_io
+        begin
+          STDOUT.reopen logfile, "a"
+          File.chmod(0644, logfile)
+          STDOUT.sync = true
+        rescue ::Exception
+          begin; STDOUT.reopen DEVNULL; rescue ::Exception; end
+        end
+        begin; STDERR.reopen STDOUT; rescue ::Exception; end
+        STDERR.sync = true
+      end
+    end
+
+    def is_respawning?; @launch_options && @launch_options['respawn']; end
+
+    def register(opt)
+      @launch_options = opt # Save for respawning
+      name_pattern = opt['worker_name']
+
+      last_wname = nil
+      widx = 0 # We do not preserve worker_index upon respawn
+      wname = nil
+      if opt['respawn']
+        wname = name_pattern
+        # Reset created for respawn purposes.
+        redis.hset(rk_worker(wname), 'created', Time.now.to_i)
+      else
+        loop do
+          wname = name_pattern.gsub('%', widx.to_s)
+          if wname == last_wname # For the name without %, we add one to the end if clashes
+            wname += widx.to_s
+            name_pattern += '%'
+          end
+
+          break if redis.hsetnx(rk_worker(wname), 'created', Time.now.to_i)
+          widx += 1
+          last_wname = wname
+        end
+      end
+      @launch_options['worker_name'] = wname # Save this in launch options
+      @launch_options['worker_index'] = widx # Sequence of the same worker
+      @launch_options['pid'] = Process.pid
+      @launch_options['machine'] = Socket.gethostname
+
+      # Setup the worker stats
+      redis.multi do
+        redis.hincrby(rk_worker, 'spawn_count', 1)
+        redis.hset(rk_worker, 'job_count', 0) # Reset the job count since respawn
+        redis.mapped_hmset(rk_worker, @launch_options) # Save launch options
+        redis.del(rk_queue) # Clear old control commands
+      end
+      nil
+    end
+
+    def unregister
+      redis.del(rk_worker)
+    end
+
+    def exists?(name)
+      redis.hexists(rk_workers, name)
+    end
+
+    def rk_worker(name = nil)
+      name ||= worker_name
+      Rescheduler.prefix + 'TMWORKER:' + name
+    end
+
+    def rk_queue(name = nil)
+      name ||= worker_name
+      Rescheduler.prefix + 'TMWORKERQUEUE:' + name
+    end
+
+    def stats
+      stats = {}
+      workers = redis.keys(rk_worker('*'))
+      kl = rk_worker('').length
+      workers.each do |w|
+        stats[w[kl..-1]] = redis.hgetall(w)
+      end
+      return stats
+    end
+
+    def named?
+      @launch_options && @launch_options.include?('worker_name')
+    end
+
+    def inc_job_count
+      # Only do this if we are launched as a worker
+      if named?
+        jc = redis.hincrby(rk_worker, 'job_count', 1)
+        # Check for respawn_jobs
+        respawn_jobs = @launch_options['respawn_jobs'].to_i
+        restart_self if respawn_jobs > 0 && jc >= respawn_jobs
+        # Check for respawn_time
+        respawn_time = @launch_options['respawn_time'].to_i
+        if respawn_time > 0
+          created = redis.hget(rk_worker, 'created')
+          restart_self if created && created.to_i + respawn_time < Time.now.to_i
+        end
+      end
+    end
+
+    def respawn_if_requested
+      return false unless @respawn
+      @launch_options['respawn'] = true
+      spawn(@launch_options)
+      return true
+    end
+
+    def spawn(opts)
+      system_options = {}
+      env = load_env(opts['env']) if opts.include?('env')
+      env ||= {}
+      #system_options['chdir'] = opts['chdir'] if opts.include?('chdir')
+
+      cmd = "rescheduler_launch #{opt_to_str(opts)}"
+      print "EXEC: #{cmd}\n"
+      pid = Kernel.spawn(env, cmd, system_options)
+      Process.detach(pid)
+    end
+
+    def load_env(env)
+      env.is_a?(String) ? Hash[env.split(';').map{|e| a,b=e.split('=', 2); b ||= true; [a,b]}] : env
+    end
+
+    def pack_env(env)
+      return env if env.is_a?(String)
+      env.map {|k,v| "#{k}=#{v}"}.join(';')
+    end
+
+    def opt_to_str(opts)
+      name = opts['worker_name']
+      file = opts['worker_file']
+      args = opts.map do |k,v|
+        next if NONCMD_OPTS.include?(k)
+        if k == 'env'
+          "--env=#{pack_env(v)}"
+        elsif v==true || v == 'true'
+          "--#{k}"
+        else
+          "--#{k}=#{v}"
+        end
+      end
+      args << name
+      args << file
+      return args.join(' ')
+    end
+
+    def redis
+      Rescheduler.send :redis # Call private method in Rescheduler module
+    end
+
+    def windows_env?; RUBY_PLATFORM.end_with?('mingw32'); end
+    def pid_exists?(pid)
+      Process.kill(0, pid.to_i)
+      return true
+    rescue Errno::ESRCH
+      return false
+    end
+  end
+end

data/lib/rescheduler.rb

@@ -1,142 +1,207 @@
+require 'date'
 require 'time' # Needed for Time.parse
 require 'multi_json'
 require 'redis'
 
+require File.expand_path('../rescheduler/worker', __FILE__)
+require File.expand_path('../rescheduler/sync', __FILE__)
+
+=begin
+
+Immediate Queue:  "TMTUBE:queue"  - List of qnids
+Deferred Tasks:   "TMDEFERRED"    - Sorted set of all tasks based on their due date
+Task Args:        "TMARGS:qnid"   - JSON args of the task
+Running Tasks:    "TMRUNNING"     - ??
+
+Maintenane:       "TMMAINT"       - ?? An internal queue for maintenance jobs
+
+Worker Semaphore: "TMWORKERLOCK"  - Exclusion semaphore for worker maintenance
+Auto-increment Id:"TMCOUNTER"     - Global unique id generator
+
+Worker Registry:  "TMWORKERS"     - Map of workerid=>worker info
+=end
+
+# NOTE: We use class variables instead of class instance variables so that
+# "include Rescheduler" would work as intended for DSL definition
+
+
 module Rescheduler
   extend self
 
   # Setup configuration
-  attr_accessor :config
-  self.config = {}
+  def config
+    @@config ||= { prefix:'' }
+    @@config
+  end
+
+  def config=(c); @@config = c; end
 
-  #==========================
-  # Management routines
+  #====================================================================
+  # Global management / Query
+  #====================================================================
   def prefix
-    return @config[:prefix] || ''
+    return @@config[:prefix]
   end
 
-  def reinitialize # Very slow reinitialize
+  def reinitialize
     keys = %w{TMCOUNTER TMMAINT TMDEFERRED TMARGS TMRUNNING}.map {|p| prefix + p }
-    %w{TMTUBE:*}.each do |p|
+    %w{TMTUBE:* TMARGS:*}.each do |p|
       keys += redis.keys(prefix + p)
     end
     redis.del(keys)
   end
 
-  # Return a hash of statistics, in this format
-  # 
+  # Warning: Linear time operation (see #show_queue)
+  def delete_queue(queue)
+    entries = show_queue(queue)
+    return 0 if entries.blank?
+
+    entries.map do |entry|
+      idelete(get_qnid(queue, entry))
+    end.length
+  end
+
+  def fast_delete_immediate_queue(queue) # NOTE: only use this when there is no inserters around
+    argkeys = redis.keys(rk_args(get_qnid(queue, '*')))
+    redis.multi do
+      redis.del(argkeys)
+      redis.del(rk_queue(queue))
+    end
+    nil
+  end
+
+  # Return a hash of statistics
   def stats
-    loop do 
-      redis.watch(rk_args) do
-        stats = {}
-        qnids = redis.hkeys(rk_args)
-        # Get all the "pending jobs"
-        qnids.each do |qnid|
-          queue = qnid_to_queue(qnid)
-          stats[queue] ||= {}
-          stats[queue][:pending] ||= 0
-          stats[queue][:pending] += 1
-        end
+    stats = {}
+    @@runners ||= {}
+    @@runners.keys.each {|queue| stats[queue] = {} } unless @@runners.blank?
+
+    # Discover all immediate queues
+    ql = rk_queue('').length
+    redis.keys(rk_queue('*')).each do |rkqueue|
+      queue = rkqueue[ql..-1]
+      stats[queue] ||= {}
+      stats[queue][:immediate] = queue_length(queue)
+    end
 
-        # Get all running
-        qnids = redis.hkeys(rk_running)
-        # Get all the "pending jobs"
-        qnids.each do |qnid|
-          queue = qnid_to_queue(qnid)
-          stats[queue] ||= {}
-          stats[queue][:running] ||= 0
-          stats[queue][:running] += 1
-        end
+    # Get all the deferred
+    deferred = redis.zrange(rk_deferred, 0, -1, :with_scores=>true)
+    deferred.each do |qnid, ts|
+      queue = qnid_to_queue(qnid)
+      stats[queue] ||= {}
+      stats[queue][:deferred] ||= 0
+      stats[queue][:deferred] += 1
+      stats[queue][:first] ||= ts # First is first
+    end
 
-        # Get all the deferred
-        deferred = redis.zrange(rk_deferred, 0, -1, :with_scores=>true)
-        deferred.each do |qnid, ts|
-          queue = qnid_to_queue(qnid)
-          stats[queue] ||= {}
-          stats[queue][:deferred] ||= 0
-          stats[queue][:deferred] += 1
-          stats[queue][:first] ||= ts # First is first
-        end        
-
-        # Get all the immediate
-        qus = stats.keys
-        quls = redis.multi do
-          qus.each { |queue| redis.llen(rk_queue(queue)) }
-        end
+    # Get all the immediate
+    return {:jobs=>stats, :workers=>Worker.stats}
+  end
 
-        unless quls # Retry
-          log_debug('Contention during stats')
-          return {:jobs=>{'Job contention'=>{}}}
-        end
+  #----------------------------------------------
+  # Queue management
+  #----------------------------------------------
+  # Returns number of jobs waiting to be handled in a queue (all immediate jobs)
+  def queue_length(queue)
+    return redis.llen(rk_queue(queue))
+  end
 
-        qus.each_with_index do |k, idx|
-          stats[k][:immediate] = quls[idx]
-        end
+  # Reads a background job and returns its properties; returns nil if the job does not exist
+  # Takes :queue and :id as arguments
+  def peek(options)
+    qnid = get_qnid(options[:queue], options[:id])
+    optstr = redis.get(rk_args(qnid))
+    return nil unless optstr
+    sopt = MultiJson.load(optstr, :symbolize_keys => true)
+    sopt[:queue] = options[:queue]
+    sopt[:id] = options[:id]
+    return sopt
+  end
 
-        return {:jobs=>stats}
-      end
-    end
+  # Warning: Linear time operation, where n is the number if items in all the queues
+  def show_queue(queue)
+    qstr = ":#{queue}:"
+    # TODO: Use SCAN after upgrade to Redis 2.8
+    redis.keys(rk_args(get_qnid(queue, '*'))).map {|k| k.split(qstr, 2).last }
   end
 
-  # NOTE: Use this with care. Some lost jobs can be moved to immediate queue instead of deleted
-  # Pass '*' to delete everything.
-  def purge_bad_jobs(queue = '*')
-    pending, running, deferred = redis.multi do 
-      redis.hkeys(rk_args)
-      redis.hkeys(rk_running)
-      redis.zrange(rk_deferred, 0, -1)
-    end
+  #----------------------------------------------
+  # Task management
+  #----------------------------------------------
 
-    bad = pending - running - deferred
-    bad.each do |qnid| 
-      next if queue != '*' && !qnid.start_with?(queue + ':')
-      idelete(qnid)
-    end
+  # Check existence of one task
+  def exists?(options)
+    raise ArgumentError, 'Can not test existence without :id' unless options.include?(:id)
+    qnid = get_qnid(options[:queue], options[:id])
+    return redis.exists(rk_args(qnid))
   end
 
-  #==========================
+  # Delete one task
+  def delete(options)
+    qnid = get_qnid(options[:queue], options[:id])
+    idelete(qnid)
+  end
+
+  #====================================================================
   # Task producer routines
+  #====================================================================
   # Add an immediate task to the queue
   def enqueue(options=nil)
-    options ||= {}
+    internal_enqueue(options, false)
+  end
+
+  def enqueue_to_top(options = nil)
+    internal_enqueue(options, true)
+  end
+
+  def internal_enqueue(options, push_to_top)
+    sopt = options ? options.dup : {}
+    queue = sopt[:queue] || '' # Default queue name is ''
+    has_id = sopt.include?(:id)
+    job_id = sopt[:id] || redis.incr(rk_counter) # Default random unique id
+
     now = Time.now.to_i
 
-    # Error check 
-    validate_queue_name(options[:queue]) if options.include?(:queue)
-    validate_recurrance(options)
+    # Error check
+    validate_queue_name(queue)
+    validate_recurrance(sopt)
 
     # Convert due_in to due_at
-    if options.include?(:due_in)
-      raise ArgumentError, ':due_in and :due_at can not be both specified' if options.include?(:due_at)
-      options[:due_at] = now + options[:due_in]
+    if sopt.include?(:due_in)
+      # log_debug 'Both due_in and due_at specified, favoring due_in' if sopt.include?(:due_at)
+      sopt[:due_at] = now + sopt[:due_in]
     end
 
-    # Get an ID if not already have one     
-    user_id = options.include?(:id)
-    unless user_id
-      options[:id] = redis.incr(rk_counter)
-    end
+    qnid = get_qnid(queue, job_id)
 
-    ts = options[:due_at].to_i
-    ts = now if ts == 0 # 0 means immediate
-    options[:due_at] = ts # Convert :due_at to integer timestamp to be reused in recurrance
-    qnid = get_qnid(options)
+    ts = sopt[:due_at].to_i
+    if ts == 0 || ts < now # immediate
+      ts = now
+      sopt.delete(:due_at)
+    else
+      raise ArgumentError, 'Can not enqueue_to_top deferred jobs' if push_to_top
+      sopt[:due_at] = ts # Convert :due_at to integer timestamp to be reused in recurrance
+    end
 
     # Encode and save args
     redis.multi do # Transaction to enqueue the job and save args together
-      if user_id # Delete possible existing job if user set id
+      if has_id # Delete possible existing job if user set id
         redis.zrem(rk_deferred, qnid)
-        redis.lrem(rk_queue(options[:queue]), 0, qnid)
+        redis.lrem(rk_queue(queue), 0, qnid) # This is going to be slow for long queues
       end
 
-      # Save options
-      redis.hset(rk_args, qnid, MultiJson.dump(options))
+      # Save args even if it is empty (for existence checks)
+      redis.set(rk_args(qnid), MultiJson.dump(sopt))
 
       # Determine the due time
       if ts > now # Future job
         redis.zadd(rk_deferred, ts, qnid)
       else
-        redis.lpush(rk_queue(options[:queue]), qnid)
+        if push_to_top
+          redis.rpush(rk_queue(queue), qnid)
+        else
+          redis.lpush(rk_queue(queue), qnid)
+        end
       end
     end
 
@@ -151,103 +216,72 @@ module Rescheduler
     nil
   end
 
-  def exists?(options)
-    raise ArgumentError, 'Can not test existence without :id' unless options.include?(:id)
-    qnid = get_qnid(options)
-    return redis.hexists(rk_args, qnid)
-  end
-
-  def enqueue_unless_exists(options)
-    enqueue(options) unless exists?(options)
-  end
-
-  def delete(options)
-    qnid = get_qnid(options)
-    idelete(qnid)
-  end
-
-  # Make a job immediate if it is not already. Erase the wait
-  def make_immediate(options)
-    dtn = rk_deferred # Make a copy in case prefix changes
-    qnid = get_qnid(options)
-    ntry = 0
-    loop do
-      redis.watch(dtn) do 
-        if redis.zcard(dtn, qnid) == 0
-          redis.unwatch(dtn)
-          return # Not a deferred job
-        else
-          redis.multi
-          redis.zrem(dtn, qnid)
-          q = qnid_to_queue(qnid)
-          redis.lpush(rk_queue(q), qnid)
-          if !redis.exec
-            # Contention happens, retrying
-            log_debug("make_immediate contention for #{qnid}")
-            Kernel.sleep (rand(ntry * 1000) / 1000.0) if ntry > 0
-          else
-            return # Done
-          end
-        end
+  # Temp function for special purpose. Completely by-pass concurrency check to increase speed
+  def quick_enqueue_batch(queue, ids, reset = false)
+    argsmap = {}
+    vals = []
+    ids.each do |id|
+      qnid = get_qnid(queue, id)
+      vals << qnid
+      argsmap[rk_args(qnid)] = '{}' # Empty args
+    end unless ids.blank?
+    redis.pipelined do # Should do redis.multi if concurrency is a problem
+      redis.del(rk_queue(queue)) if reset # Empty the list fast
+      unless ids.blank?
+        redis.lpush(rk_queue(queue), vals)
+        argsmap.each { |k,v| redis.set(k,v) }
       end
-      ntry += 1
     end
+    nil
   end
 
-  #=================
-  # Serialization (in case it is needed to transfer all Rescheduler across to another redis instance)
-
-  # Atomically save the state to file and stop all workers (state in redis is not destroyed)
-  # This function can take a while as it will wait for running jobs to finish first.
-  def serialize_and_stop(filename)
-    # TODO
-  end
-
-  # Load state from a file. Will merge into existing jobs if there are any (make sure it is done only once)
-  # This can be done before any worker starts, or after. 
-  # Workers still need to be manually started
-  def deserialize(filename)
-    # TODO
-  end
-
-  # Clear redis states and delete all jobs (useful before deserialize)
-  def erase_all
-    # TODO
+  # Returns true if enqueued a new job, otherwise returns false
+  def enqueue_unless_exists(options)
+    # NOTE: There is no point synchronizing exists and enqueue
+    return false if exists?(options)
+    enqueue(options)
+    return true
   end
 
-  #=================
+  #====================================================================
   # Job definition
+  #====================================================================
+
   # Task consumer routines
   def job(tube, &block)
-    @runners ||= {}
-    @runners[tube] = block
+    @@runners ||= {}
+    @@runners[tube] = block
     return nil
   end
 
-  #=================
+  #====================================================================
   # Error handling
+  #====================================================================
   def on_error(tube=nil, &block)
     if tube != nil
-      @error_handlers ||= {}
-      @error_handlers[tube] = block
+      @@error_handlers ||= {}
+      @@error_handlers[tube] = block
     else
-      @global_error_handler = block;
+      @@global_error_handler = block;
     end
   end
-  #=================
+
+  #====================================================================
   # Runner/Maintenance routines
+  #====================================================================
   def start(*tubes)
+    @@runners ||= {}
     # Check arguments
-    if !@runners || @runners.size == 0
+    if @@runners.size == 0
       raise Exception, 'Can not start worker without defining job handlers.'
     end
 
     tubes.each do |t|
-      next if @runners.include?(t)
+      next if @@runners.include?(t)
       raise Exception, "Handler for queue #{t} is undefined."
     end
 
-    tubes = @runners.keys if !tubes || tubes.size == 0
+    tubes = @@runners.keys if !tubes || tubes.size == 0
 
     log_debug "[[ Starting: #{tubes.join(',')} ]]"
 
@@ -257,20 +291,33 @@ module Rescheduler
     keys = tubes.map {|t| rk_queue(t)}
     keys << rk_maintenace
 
+    # Queue to control a named worker
+    worker_queue = Worker.rk_queue if Worker.named?
+    keys.unshift(worker_queue) if worker_queue # worker control queue is the first we respond to
+
     dopush = nil
 
-    loop do
+    @@end_job_loop = false
+    while !@@end_job_loop
       # Run maintenance and determine timeout
       next_job_time = determine_next_deferred_job_time.to_i
 
       if dopush # Only pass-on the token after we are done with maintenance. Avoid contention
-        redis.lpush(rk_maintenace, dopush) 
+        redis.lpush(rk_maintenace, dopush)
         dopush = nil
       end
 
       # Blocking wait
       timeout = next_job_time - Time.now.to_i
       timeout = 1 if timeout < 1
+
+      # A producer may insert another job after BRPOP and before WATCH
+      # Due to limitations of BRPOP we can not prevent this from happening.
+      # When it happens we will consume the args of the later job, causing
+      # the newly inserted job to be "promoted" to the front of the queue
+      # This may not be desirable...
+      # (too bad BRPOPLPUSH does not support multiple queues...)
+      # TODO: Maybe LUA script is the way out of this.
       result = redis.brpop(keys, :timeout=>timeout)
 
       # Handle task
@@ -278,25 +325,39 @@ module Rescheduler
         tube = result[0]
         qnid = result[1]
         if tube == rk_maintenace
-          # Circulate maintenance task until it comes a full circle. This depends on redis 
-          # first come first serve policy in brpop. 
+          # Circulate maintenance task until it comes a full circle. This depends on redis
+          # first come first serve policy in brpop.
           dopush = qnid + client_key unless qnid.include?(client_key) # Push if we have not pushed yet.
+        elsif tube == worker_queue
+          Worker.handle_command(qnid)
         else
           run_job(qnid)
         end
-      else 
+      else
         # Do nothing when got timeout, the run_maintenance will take care of deferred jobs
       end
     end
   end
 
-  private 
+  def end_job_loop; @@end_job_loop = true; end
+
+
+  # Logging facility
+  def log_debug(msg)
+    return if config[:silent]
+    print("#{Time.now.iso8601} #{msg}\n")
+    STDOUT.flush
+  end
+
+  #====================================================================
+  private
+  #====================================================================
 
   # Internal routines operating out of qnid
   def idelete(qnid)
     queue = qnid.split(':').first
     redis.multi do
-      redis.hdel(rk_args, qnid)
+      redis.del(rk_args(qnid))
       redis.zrem(rk_deferred, qnid)
       redis.lrem(rk_queue(queue), 0, qnid)
     end
@@ -306,74 +367,70 @@ module Rescheduler
   def run_job(qnid)
     # 1. load job parameters for running
     optstr = nil
-    begin
-      res = nil
-      # Note: We use a single key to watch, can be improved by having a per-job key, 
-      redis.watch(rk_args) do # Transaction to ensure read/delete is atomic
-        optstr = redis.hget(rk_args, qnid)        
-        if optstr.nil?
-          redis.unwatch
-          log_debug("Job is deleted mysteriously")
-          return # Job is deleted somewhere
-        end
-        res = redis.multi do 
-          redis.hdel(rk_args, qnid)
-          redis.hset(rk_running, qnid, optstr)
-        end
-        if !res
-          # Contention, try read again
-          log_debug("Job read contention: (#{qnid})")
-        end
-      end
-    end until res
+    key = rk_args(qnid)
+    # Atomic get and delete the arg
+    redis.multi do
+      optstr = redis.get(key)
+      redis.del(key)
+    end
+
+    optstr = optstr.value # get the value from the multi block future
+    if optstr.nil?
+      log_debug("Job is deleted mysteriously: (#{qnid})")
+      return # Job is deleted somewhere
+    end
 
     # Parse and run
     sopt = MultiJson.load(optstr, :symbolize_keys => true)
+    queue,id = qnid.split(':', 2)
+    sopt[:queue] ||= queue
+    sopt[:id] ||= id
 
     # Handle parameters
     if (sopt.include?(:recur_every))
       newopt = sopt.dup
       newopt[:due_at] = (sopt[:due_at] || Time.now).to_i + sopt[:recur_every].to_i
       newopt.delete(:due_in) # In case the first job was specified by :due_in
-      log_debug("---Enqueue #{qnid}: due_every #{sopt[:due_every]}")
+      log_debug("---Enqueue #{qnid}: recur_every #{sopt[:recur_every]}")
       enqueue(newopt)
     end
 
-    if (sopt.include?(:recur_daily))
-      newopt = sopt.dup      
-      newopt[:due_at] = time_from_recur_daily(sopt[:recur_daily])
-      newopt.delete(:due_in) # In case the first job was specified by :due_in
-      log_debug("---Enqueue #{qnid}: due_daily #{sopt[:recur_daily]}")
+    if (sopt.include?(:recur_daily) || sopt.include?(:recur_weekly))
+      newopt = sopt.dup
+      newopt.delete(:due_at)
+      newopt.delete(:due_in) # No more due info, just the recurrance
+      log_debug("---Enqueue #{qnid}: recur_daily #{sopt[:recur_daily]}") if sopt.include?(:recur_daily)
+      log_debug("---Enqueue #{qnid}: recur_weekly #{sopt[:recur_weekly]}") if sopt.include?(:recur_weekly)
       enqueue(newopt)
     end
 
     # 2. Find runner and invoke it
     begin
       log_debug(">>---- Starting #{qnid}")
-      runner = @runners[qnid_to_queue(qnid)]
+      runner = @@runners[qnid_to_queue(qnid)]
       if runner.is_a?(Proc)
         runner.call(sopt)
         log_debug("----<< Finished #{qnid}")
+        Worker.inc_job_count # Stats for the worker
       else
         log_debug("----<< Failed #{qnid}: Unknown queue name, handler not defined")
       end
     rescue Exception => e
       log_debug("----<< Failed #{qnid}: -------------\n #{$!}")
       log_debug(e.backtrace[0..4].join("\n"))
-      handle_error(e, qnid, sopt)
+      handle_error(e, queue, sopt)
       log_debug("------------------------------------\n")
     end
-
-    # 3. Remove job from running list (Done)
-    redis.hdel(rk_running, qnid)
   end
 
-  def handle_error(e, qnid, sopt)
-    error_handler = @error_handlers && @error_handlers[qnid]
+  def handle_error(e, queue, sopt)
+    @@error_handlers ||= {}
+    @@global_error_handler ||= nil
+    error_handler = @@error_handlers[queue]
     if error_handler
-      error_handler.call(e, sopt) 
-    elsif @global_error_handler
-      @global_error_handler.call(e, sopt) 
+      error_handler.call(e, sopt)
+    elsif @@global_error_handler
+      @@global_error_handler.call(e, sopt)
     end
   end
 
@@ -383,17 +440,18 @@ module Rescheduler
   def service_deferred_jobs
     dtn = rk_deferred # Make a copy in case prefix changes
     ntry = 0
-    loop do
+    while ntry < 6 do
       curtime = Time.now.to_i
+      return if redis.zcount(dtn, 0, curtime) == 0
+
+      limit = ntry < 3 ? 100 : 1 # After first 3 tries, do just 1
       redis.watch(dtn) do
-        tasks = redis.zrangebyscore(dtn, 0, curtime)
+        tasks = redis.zrangebyscore(dtn, 0, curtime, :limit=>[0,limit]) # Serve at most 100
         if tasks.empty?
           redis.unwatch
           return # Nothing to transfer, moving on.
         end
 
-        redis.multi
-        redis.zremrangebyscore(dtn, 0, curtime)
         to_push = {}
         tasks.each do |qnid|
           q = rk_queue(qnid_to_queue(qnid))
@@ -401,6 +459,9 @@ module Rescheduler
           to_push[q] << qnid
         end
 
+        redis.multi
+        redis.zrem(dtn, tasks)
+
         to_push.each do |q, qnids|
           redis.lpush(q, qnids) # Batch command
         end
@@ -409,63 +470,30 @@ module Rescheduler
           # Contention happens, retrying
           # Sleep a random amount of time after first try
           ntry += 1
-          log_debug("service_deferred_jobs contention")
+          log_debug("service_deferred_jobs(#{limit}) contention")
           Kernel.sleep (rand(ntry * 1000) / 1000.0)
         else
           return # Done transfering
         end
       end
-
-      if ntry > 3 # Max number of tries
-        # Fall back to 
-        service_one_deferred_job
-        return
-      end
     end
-  end
-
-  def service_one_deferred_jobs
-    dtn = rk_deferred # Make a copy in case prefix changes
-    ntry = 0
-    curtime = Time.now.to_i
-    loop do 
-      redis.watch(dtn) do
-        tasks = redis.zrangebyscore(dtn, 0, curtime, :limit=>[0,1])
-        if tasks.empty?
-          redis.unwatch
-          return # Nothing to transfer, moving on.
-        end
 
-        qnid = tasks[0]
-        q = qnid_to_queue(qnid)
-
-        redis.multi
-        redis.zrem(dtn, qnid)
-        redis.lpush(rk_queue(q), qnid)
-        if !redis.exec
-          # Contention happens, retrying
-          # Sleep a random amount of time after first try
-          log_debug("service_one_deferred_job contention")
-          ntry += 1
-          Kernel.sleep (rand(ntry * 1000) / 1000.0)
-        else
-          break # Done transfering one job
-        end
-      end
-    end
+    log_debug("service_deferred_jobs failed, will try next time")
   end
 
   def determine_next_deferred_job_time(skip_service = nil)
     tsnow = Time.now.to_f
-    maxtime = tsnow + 3600
+    maxtime = tsnow + 3600 + rand(100) # Randomize wake time to avoid multi worker service contention
 
     dt = redis.zrange(rk_deferred, 0, 0, :with_scores=>true)[0]
     nt = (dt && dt[1] && dt[1] < maxtime) ? dt[1] : maxtime
     if !skip_service && nt <= tsnow
-      service_deferred_jobs
-      # Get the deferred jobs again.
-      dt = redis.zrange(rk_deferred, 0, 0, :with_scores=>true)[0]
-      nt = (dt && dt[1] && dt[1] < maxtime) ? dt[1] : maxtime
+      do_if_can_acquire_semaphore do
+        service_deferred_jobs
+        # Get the deferred jobs again.
+        dt = redis.zrange(rk_deferred, 0, 0, :with_scores=>true)[0]
+        nt = (dt && dt[1] && dt[1] < maxtime) ? dt[1] : maxtime
+      end
     end
     return nt
   end
@@ -474,50 +502,99 @@ module Rescheduler
 
   def rk_deferred; prefix + 'TMDEFERRED'; end
   def rk_maintenace; prefix + 'TMMAINT'; end
-  def rk_args; prefix + "TMARGS"; end
-  def rk_running; prefix + "TMRUNNING"; end
+  def rk_args(qnid); "#{prefix}TMARGS:#{qnid}"; end
   def rk_counter; prefix + 'TMCOUNTER'; end
+  def rk_worker_semaphore; prefix + 'TMWORKERLOCK'; end # This is a boolean with a timeout for workers to exclude each other
+
+  # None blocking, returns true if semaphore is acquired (for a given timeout), this is cooperative to avoid guaranteed contentions
+  def try_acquire_semaphore(timeout=300) # Default for 5 minutes, there must be a timeout
+    semkey = rk_worker_semaphore
+    if redis.setnx(semkey, 1) # Any value would be fine
+      redis.expire(semkey, timeout)
+      return true
+    else
+      # Already created, someone has it
+      return false
+    end
+  end
+
+  # This releases semaphore unconditionally
+  def release_semaphore
+    # NOTE: There is a chance we remove the lock created by another worker after our own expired.
+    # This is OK since the lock is cooperative and not necessary (real locking is done through contention checks)
+    redis.del(rk_worker_semaphore)
+  end
 
-  def get_qnid(options)
-    return "#{options[:queue]}:#{options[:id]}"
+  # Run block only if the semaphore can be acquired, otherwise do nothing
+  def do_if_can_acquire_semaphore
+    if try_acquire_semaphore
+      yield
+      release_semaphore
+    end
   end
 
+  def get_qnid(queue, id); return "#{queue}:#{id}"; end
+
   def qnid_to_queue(qnid)
     idx = qnid.index(':')
     unless idx
       log_debug("Invalid qnid: #{qnid}")
-      return nil 
+      return nil
     end
     qnid[0...idx]
   end
 
   def redis
-    @redis ||= @config[:redis] || Redis.new(@config[:redis_connection] || {})
+    @@redis ||= config[:redis] || Redis.new(config[:redis_connection] || {})
   end
 
   def validate_queue_name(queue)
     raise ArgumentError, 'Queue name can not contain special characters' if queue.include?(':')
   end
 
-  def parse_seconds_of_day(recur_daily)
-    return recur_daily if recur_daily.is_a?(Fixnum)
-    time = Time.parse(recur_daily)
-    return time.to_i - Time.new(time.year, time.month, time.day).to_i
-  end
-
   # Find the next recur time
   def time_from_recur_daily(recur_daily, now = Time.now)
-    recur = parse_seconds_of_day(recur_daily)
-    t = Time.new(now.year, now.month, now.day).to_i + recur
-    t += 86400 if t < now.to_i
-    return Time.at(t)
+    parsed = Date._parse(recur_daily)
+    if !parsed[:hour] || (parsed.keys - [:zone, :hour, :min, :sec, :offset, :sec_fraction]).present?
+      raise ArgumentError, 'Unexpected recur_daily value: ' + recur_daily
+    end
+
+    if !parsed[:offset]
+      raise ArgumentError, 'A timezone is required for recur_daily: ' + recur_daily
+    end
+
+    # Never offset over one day (e.g. 23:59 PDT)
+    offset = (parsed[:hour] * 3600 + (parsed[:min]||0) * 60 + (parsed[:sec] || 0) - parsed[:offset]) % 86400
+
+    t = Time.utc(now.year, now.month, now.day) + offset
+    t += 86400 if t <= now + 1
+    return t
+  end
+
+  def time_from_recur_weekly(recur_weekly, now = Time.now)
+    parsed = Date._parse(recur_weekly)
+    if !parsed[:hour] || !parsed[:wday] || (parsed.keys - [:wday, :zone, :hour, :min, :sec, :offset, :sec_fraction]).present?
+      raise ArgumentError, 'Unexpected recur_weekly value: ' + recur_weekly
+    end
+
+    if !parsed[:offset]
+      raise ArgumentError, 'A timezone is required for recur_weekly: ' + recur_weekly
+    end
+
+    # Never offset over one week
+    offset = parsed[:hour] * 3600 + (parsed[:min]||0) * 60 + (parsed[:sec] || 0) - parsed[:offset]
+    offset = (offset + parsed[:wday] * 86400) % (86400 * 7)
+
+    t = Time.utc(now.year, now.month, now.day) - now.wday * 86400 + offset
+    t += 86400 * 7 if t <= now + 1
+    return t
   end
 
   def validate_recurrance(options)
     rcnt = 0
     if (options.include?(:recur_every))
       rcnt += 1
-      raise 'Expect integer for :recur_every parameter' unless options[:recur_every].is_a?(Fixnum)
+      raise 'Expect integer for :recur_every parameter' unless options[:recur_every].is_a?(Integer)
     end
 
     if (options.include?(:recur_daily))
@@ -527,14 +604,16 @@ module Rescheduler
         options[:due_at] = time # Setup the first run
       end
     end
-    raise 'Can only specify one recurrance parameter' if rcnt > 1
-  end
 
-  # Logging facility
-  def log_debug(msg)
-    return if @config[:silent]
-    print("#{Time.now.iso8601} #{msg}\n")
-    STDOUT.flush
+    if (options.include?(:recur_weekly))
+      rcnt += 1
+      time = time_from_recur_weekly(options[:recur_weekly]) # Try parse and make sure we can
+      unless options.include?(:due_at) || options.include?(:due_in)
+        options[:due_at] = time # Setup the first run
+      end
+    end
+
+    raise 'Can only specify one recurrance parameter' if rcnt > 1
   end
 
 end

metadata

@@ -1,80 +1,77 @@
 --- !ruby/object:Gem::Specification
 name: rescheduler
 version: !ruby/object:Gem::Version
-  version: 0.4.1
-  prerelease: 
+  version: 0.5.1
 platform: ruby
 authors:
 - Dongyi Liao
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-22 00:00:00.000000000 Z
+date: 2016-06-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redis
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Rescheduler is a library that uses Redis to maintain a task queue of
   immediate and delayed jobs without polling.
 email: liaody@gmail.com
-executables: []
+executables:
+- rescheduler_launch
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/rescheduler_launch
 - lib/rescheduler.rb
+- lib/rescheduler/sync.rb
+- lib/rescheduler/worker.rb
 homepage: https://github.com/liaody/rescheduler
 licenses:
 - BSD
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.5.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A job queue for immediate and delayed jobs using Redis
 test_files: []
-has_rdoc: