resque-admin-scheduler 1.0.2 → 1.3.0

data/lib/resque_admin/scheduler/delaying_extensions.rb

@@ -0,0 +1,324 @@
+# vim:fileencoding=utf-8
+require 'resque_admin'
+require_relative 'plugin'
+require_relative '../scheduler'
+
+module ResqueAdmin
+  module Scheduler
+    module DelayingExtensions
+      # This method is nearly identical to +enqueue+ only it also
+      # takes a timestamp which will be used to schedule the job
+      # for queueing.  Until timestamp is in the past, the job will
+      # sit in the schedule list.
+      def enqueue_at(timestamp, klass, *args)
+        validate(klass)
+        enqueue_at_with_queue(
+          queue_from_class(klass), timestamp, klass, *args
+        )
+      end
+
+      # Identical to +enqueue_at+, except you can also specify
+      # a queue in which the job will be placed after the
+      # timestamp has passed. It respects ResqueAdmin.inline option, by
+      # creating the job right away instead of adding to the queue.
+      def enqueue_at_with_queue(queue, timestamp, klass, *args)
+        return false unless plugin.run_before_schedule_hooks(klass, *args)
+
+        if ResqueAdmin.inline? || timestamp.to_i < Time.now.to_i
+          # Just create the job and let resque_admin perform it right away with
+          # inline.  If the class is a custom job class, call self#scheduled
+          # on it. This allows you to do things like
+          # ResqueAdmin.enqueue_at(timestamp, CustomJobClass, :opt1 => val1).
+          # Otherwise, pass off to ResqueAdmin.
+          if klass.respond_to?(:scheduled)
+            klass.scheduled(queue, klass.to_s, *args)
+          else
+            ResqueAdmin::Job.create(queue, klass, *args)
+          end
+        else
+          delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
+        end
+
+        plugin.run_after_schedule_hooks(klass, *args)
+      end
+
+      # Identical to enqueue_at but takes number_of_seconds_from_now
+      # instead of a timestamp.
+      def enqueue_in(number_of_seconds_from_now, klass, *args)
+        enqueue_at(Time.now + number_of_seconds_from_now, klass, *args)
+      end
+
+      # Identical to +enqueue_in+, except you can also specify
+      # a queue in which the job will be placed after the
+      # number of seconds has passed.
+      def enqueue_in_with_queue(queue, number_of_seconds_from_now,
+                                klass, *args)
+        enqueue_at_with_queue(queue, Time.now + number_of_seconds_from_now,
+                              klass, *args)
+      end
+
+      # Used internally to stuff the item into the schedule sorted list.
+      # +timestamp+ can be either in seconds or a datetime object Insertion
+      # if O(log(n)).  Returns true if it's the first job to be scheduled at
+      # that time, else false
+      def delayed_push(timestamp, item)
+        # First add this item to the list for this timestamp
+        redis.rpush("delayed:#{timestamp.to_i}", encode(item))
+
+        # Store the timestamps at with this item occurs
+        redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
+
+        # Now, add this timestamp to the zsets.  The score and the value are
+        # the same since we'll be querying by timestamp, and we don't have
+        # anything else to store.
+        redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
+      end
+
+      # Returns an array of timestamps based on start and count
+      def delayed_queue_peek(start, count)
+        result = redis.zrange(:delayed_queue_schedule, start,
+                              start + count - 1)
+        Array(result).map(&:to_i)
+      end
+
+      # Returns the size of the delayed queue schedule
+      def delayed_queue_schedule_size
+        redis.zcard :delayed_queue_schedule
+      end
+
+      # Returns the number of jobs for a given timestamp in the delayed queue
+      # schedule
+      def delayed_timestamp_size(timestamp)
+        redis.llen("delayed:#{timestamp.to_i}").to_i
+      end
+
+      # Returns an array of delayed items for the given timestamp
+      def delayed_timestamp_peek(timestamp, start, count)
+        if 1 == count
+          r = list_range "delayed:#{timestamp.to_i}", start, count
+          r.nil? ? [] : [r]
+        else
+          list_range "delayed:#{timestamp.to_i}", start, count
+        end
+      end
+
+      # Returns the next delayed queue timestamp
+      # (don't call directly)
+      def next_delayed_timestamp(at_time = nil)
+        search_first_delayed_timestamp_in_range(nil, at_time || Time.now)
+      end
+
+      # Returns the next item to be processed for a given timestamp, nil if
+      # done. (don't call directly)
+      # +timestamp+ can either be in seconds or a datetime
+      def next_item_for_timestamp(timestamp)
+        key = "delayed:#{timestamp.to_i}"
+
+        encoded_item = redis.lpop(key)
+        redis.srem("timestamps:#{encoded_item}", key)
+        item = decode(encoded_item)
+
+        # If the list is empty, remove it.
+        clean_up_timestamp(key, timestamp)
+        item
+      end
+
+      # Clears all jobs created with enqueue_at or enqueue_in
+      def reset_delayed_queue
+        Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |item|
+          key = "delayed:#{item}"
+          items = redis.lrange(key, 0, -1)
+          redis.pipelined do
+            items.each { |ts_item| redis.del("timestamps:#{ts_item}") }
+          end
+          redis.del key
+        end
+
+        redis.del :delayed_queue_schedule
+      end
+
+      # Given an encoded item, remove it from the delayed_queue
+      def remove_delayed(klass, *args)
+        search = encode(job_to_hash(klass, args))
+        remove_delayed_job(search)
+      end
+
+      # Given an encoded item, enqueue it now
+      def enqueue_delayed(klass, *args)
+        hash = job_to_hash(klass, args)
+        remove_delayed(klass, *args).times do
+          ResqueAdmin::Scheduler.enqueue_from_config(hash)
+        end
+      end
+
+      # Given a block, remove jobs that return true from a block
+      #
+      # This allows for removal of delayed jobs that have arguments matching
+      # certain criteria
+      def remove_delayed_selection(klass = nil)
+        raise ArgumentError, 'Please supply a block' unless block_given?
+
+        found_jobs = find_delayed_selection(klass) { |args| yield(args) }
+        found_jobs.reduce(0) do |sum, encoded_job|
+          sum + remove_delayed_job(encoded_job)
+        end
+      end
+
+      # Given a block, enqueue jobs now that return true from a block
+      #
+      # This allows for enqueuing of delayed jobs that have arguments matching
+      # certain criteria
+      def enqueue_delayed_selection(klass = nil)
+        raise ArgumentError, 'Please supply a block' unless block_given?
+
+        found_jobs = find_delayed_selection(klass) { |args| yield(args) }
+        found_jobs.reduce(0) do |sum, encoded_job|
+          decoded_job = decode(encoded_job)
+          klass = Util.constantize(decoded_job['class'])
+          sum + enqueue_delayed(klass, *decoded_job['args'])
+        end
+      end
+
+      # Given a block, find jobs that return true from a block
+      #
+      # This allows for finding of delayed jobs that have arguments matching
+      # certain criteria
+      def find_delayed_selection(klass = nil, &block)
+        raise ArgumentError, 'Please supply a block' unless block_given?
+
+        timestamps = redis.zrange(:delayed_queue_schedule, 0, -1)
+
+        # Beyond 100 there's almost no improvement in speed
+        found = timestamps.each_slice(100).map do |ts_group|
+          jobs = redis.pipelined do |r|
+            ts_group.each do |ts|
+              r.lrange("delayed:#{ts}", 0, -1)
+            end
+          end
+
+          jobs.flatten.select do |payload|
+            payload_matches_selection?(decode(payload), klass, &block)
+          end
+        end
+
+        found.flatten
+      end
+
+      # Given a timestamp and job (klass + args) it removes all instances and
+      # returns the count of jobs removed.
+      #
+      # O(N) where N is the number of jobs scheduled to fire at the given
+      # timestamp
+      def remove_delayed_job_from_timestamp(timestamp, klass, *args)
+        return 0 if ResqueAdmin.inline?
+
+        key = "delayed:#{timestamp.to_i}"
+        encoded_job = encode(job_to_hash(klass, args))
+
+        redis.srem("timestamps:#{encoded_job}", key)
+        count = redis.lrem(key, 0, encoded_job)
+        clean_up_timestamp(key, timestamp)
+
+        count
+      end
+
+      def count_all_scheduled_jobs
+        total_jobs = 0
+        Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |ts|
+          total_jobs += redis.llen("delayed:#{ts}").to_i
+        end
+        total_jobs
+      end
+
+      # Discover if a job has been delayed.
+      # Examples
+      #   ResqueAdmin.delayed?(MyJob)
+      #   ResqueAdmin.delayed?(MyJob, id: 1)
+      # Returns true if the job has been delayed
+      def delayed?(klass, *args)
+        !scheduled_at(klass, *args).empty?
+      end
+
+      # Returns delayed jobs schedule timestamp for +klass+, +args+.
+      def scheduled_at(klass, *args)
+        search = encode(job_to_hash(klass, args))
+        redis.smembers("timestamps:#{search}").map do |key|
+          key.tr('delayed:', '').to_i
+        end
+      end
+
+      def last_enqueued_at(job_name, date)
+        redis.hset('delayed:last_enqueued_at', job_name, date)
+      end
+
+      def get_last_enqueued_at(job_name)
+        redis.hget('delayed:last_enqueued_at', job_name)
+      end
+
+      private
+
+      def job_to_hash(klass, args)
+        { class: klass.to_s, args: args, queue: queue_from_class(klass) }
+      end
+
+      def job_to_hash_with_queue(queue, klass, args)
+        { class: klass.to_s, args: args, queue: queue }
+      end
+
+      def remove_delayed_job(encoded_job)
+        return 0 if ResqueAdmin.inline?
+
+        timestamps = redis.smembers("timestamps:#{encoded_job}")
+
+        replies = redis.pipelined do
+          timestamps.each do |key|
+            redis.lrem(key, 0, encoded_job)
+            redis.srem("timestamps:#{encoded_job}", key)
+          end
+        end
+
+        return 0 if replies.nil? || replies.empty?
+        replies.each_slice(2).map(&:first).inject(:+)
+      end
+
+      def clean_up_timestamp(key, timestamp)
+        # Use a watch here to ensure nobody adds jobs to this delayed
+        # queue while we're removing it.
+        redis.watch(key) do
+          if redis.llen(key).to_i == 0
+            # If the list is empty, remove it.
+            redis.multi do
+              redis.del(key)
+              redis.zrem(:delayed_queue_schedule, timestamp.to_i)
+            end
+          else
+            redis.redis.unwatch
+          end
+        end
+      end
+
+      def search_first_delayed_timestamp_in_range(start_at, stop_at)
+        start_at = start_at.nil? ? '-inf' : start_at.to_i
+        stop_at = stop_at.nil? ? '+inf' : stop_at.to_i
+
+        items = redis.zrangebyscore(
+          :delayed_queue_schedule, start_at, stop_at,
+          limit: [0, 1]
+        )
+        timestamp = items.nil? ? nil : Array(items).first
+        timestamp.to_i unless timestamp.nil?
+      end
+
+      def payload_matches_selection?(decoded_payload, klass)
+        return false if decoded_payload.nil?
+        job_class = decoded_payload['class']
+        relevant_class = (klass.nil? || klass.to_s == job_class)
+        relevant_class && yield(decoded_payload['args'])
+      end
+
+      def plugin
+        ResqueAdmin::Scheduler::Plugin
+      end
+    end
+  end
+end

data/lib/resque_admin/scheduler/env.rb

@@ -0,0 +1,89 @@
+# vim:fileencoding=utf-8
+
+require 'English' # $PROCESS_ID
+
+module ResqueAdmin
+  module Scheduler
+    class Env
+      def initialize(options)
+        @options = options
+        @pidfile_path = nil
+      end
+
+      def setup
+        require 'resque_admin'
+        require 'resque_admin/scheduler'
+
+        setup_backgrounding
+        setup_pid_file
+        setup_scheduler_configuration
+      end
+
+      def cleanup
+        cleanup_pid_file
+      end
+
+      private
+
+      attr_reader :options, :pidfile_path
+
+      def setup_backgrounding
+        return unless options[:background]
+
+        # Need to set this here for conditional Process.daemon redirect of
+        # stderr/stdout to /dev/null
+        ResqueAdmin::Scheduler.quiet = if options.key?(:quiet)
+                                    !!options[:quiet]
+                                  else
+                                    true
+                                  end
+
+        unless Process.respond_to?('daemon')
+          abort 'background option is set, which requires ruby >= 1.9'
+        end
+
+        Process.daemon(true, !ResqueAdmin::Scheduler.quiet)
+        ResqueAdmin.redis.client.reconnect
+      end
+
+      def setup_pid_file
+        return unless options[:pidfile]
+
+        @pidfile_path = File.expand_path(options[:pidfile])
+
+        File.open(pidfile_path, 'w') do |f|
+          f.puts $PROCESS_ID
+        end
+
+        at_exit { cleanup_pid_file }
+      end
+
+      def setup_scheduler_configuration
+        ResqueAdmin::Scheduler.configure do |c|
+          c.app_name = options[:app_name] if options.key?(:app_name)
+
+          c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
+
+          c.env = options[:env] if options.key(:env)
+
+          c.logfile = options[:logfile] if options.key?(:logfile)
+
+          c.logformat = options[:logformat] if options.key?(:logformat)
+
+          if psleep = options[:poll_sleep_amount] && !psleep.nil?
+            c.poll_sleep_amount = Float(psleep)
+          end
+
+          c.verbose = !!options[:verbose] if options.key?(:verbose)
+        end
+      end
+
+      def cleanup_pid_file
+        return unless pidfile_path
+
+        File.delete(pidfile_path) if File.exist?(pidfile_path)
+        @pidfile_path = nil
+      end
+    end
+  end
+end

data/lib/{resque → resque_admin}/scheduler/extension.rb

@@ -3,7 +3,7 @@
 require_relative 'scheduling_extensions'
 require_relative 'delaying_extensions'
 
-module Resque
+module ResqueAdmin
   module Scheduler
     module Extension
       include SchedulingExtensions

data/lib/{resque → resque_admin}/scheduler/failure_handler.rb

@@ -1,8 +1,8 @@
-module Resque
+module ResqueAdmin
   module Scheduler
     class FailureHandler
       def self.on_enqueue_failure(_, e)
-        Resque::Scheduler.log_error(
+        ResqueAdmin::Scheduler.log_error(
           "#{e.class.name}: #{e.message} #{e.backtrace.inspect}"
         )
       end

data/lib/{resque → resque_admin}/scheduler/lock/base.rb

@@ -1,6 +1,6 @@
 # vim:fileencoding=utf-8
 
-module Resque
+module ResqueAdmin
   module Scheduler
     module Lock
       class Base
@@ -30,7 +30,7 @@ module Resque
 
         # Releases the lock.
         def release!
-          Resque.redis.del(key) == 1
+          ResqueAdmin.redis.del(key) == 1
         end
 
         # Releases the lock iff we own it
@@ -42,7 +42,7 @@ module Resque
 
         # Extends the lock by `timeout` seconds.
         def extend_lock!
-          Resque.redis.expire(key, timeout)
+          ResqueAdmin.redis.expire(key, timeout)
         end
 
         def hostname

data/lib/{resque → resque_admin}/scheduler/lock/basic.rb

@@ -1,22 +1,22 @@
 # vim:fileencoding=utf-8
 require_relative 'base'
 
-module Resque
+module ResqueAdmin
   module Scheduler
     module Lock
       class Basic < Base
         def acquire!
-          if Resque.redis.setnx(key, value)
+          if ResqueAdmin.redis.setnx(key, value)
             extend_lock!
             true
           end
         end
 
         def locked?
-          if Resque.redis.get(key) == value
+          if ResqueAdmin.redis.get(key) == value
             extend_lock!
 
-            return true if Resque.redis.get(key) == value
+            return true if ResqueAdmin.redis.get(key) == value
           end
 
           false

data/lib/{resque → resque_admin}/scheduler/lock/resilient.rb

@@ -1,7 +1,7 @@
 # vim:fileencoding=utf-8
 require_relative 'base'
 
-module Resque
+module ResqueAdmin
   module Scheduler
     module Lock
       class Resilient < Base
@@ -26,7 +26,7 @@ module Resque
 
         def evalsha(script, keys, argv, refresh: false)
           sha_method_name = "#{script}_sha"
-          Resque.redis.evalsha(
+          ResqueAdmin.redis.evalsha(
             send(sha_method_name, refresh),
             keys: keys,
             argv: argv
@@ -43,7 +43,7 @@ module Resque
           @locked_sha = nil if refresh
 
           @locked_sha ||=
-            Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+            ResqueAdmin.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
               if redis.call('GET', KEYS[1]) == ARGV[1]
               then
                 redis.call('EXPIRE', KEYS[1], #{timeout})
@@ -62,7 +62,7 @@ module Resque
           @acquire_sha = nil if refresh
 
           @acquire_sha ||=
-            Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+            ResqueAdmin.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
               if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
               then
                 redis.call('EXPIRE', KEYS[1], #{timeout})

data/lib/resque_admin/scheduler/lock.rb

@@ -0,0 +1,4 @@
+# vim:fileencoding=utf-8
+%w(base basic resilient).each do |file|
+  require "resque_admin/scheduler/lock/#{file}"
+end

data/lib/resque_admin/scheduler/locking.rb

@@ -0,0 +1,104 @@
+# vim:fileencoding=utf-8
+
+# ### Locking the scheduler process
+#
+# There are two places in resque_admin-scheduler that need to be synchonized in order
+# to be able to run redundant scheduler processes while ensuring jobs don't get
+# queued multiple times when the master process changes.
+#
+# 1) Processing the delayed queues (jobs that are created from
+# enqueue_at/enqueue_in, etc) 2) Processing the scheduled (cron-like) jobs from
+# rufus-scheduler
+#
+# Protecting the delayed queues (#1) is relatively easy.  A simple SETNX in
+# redis would suffice.  However, protecting the scheduled jobs is trickier
+# because the clocks on machines could be slightly off or actual firing times
+# could vary slightly due to load.  If scheduler A's clock is slightly ahead of
+# scheduler B's clock (since they are on different machines), when scheduler A
+# dies, we need to ensure that scheduler B doesn't queue jobs that A already
+# queued before it's death. (This all assumes that it is better to miss a few
+# scheduled jobs than it is to run them multiple times for the same iteration.)
+#
+# To avoid queuing multiple jobs in the case of master fail-over, the master
+# should remain the master as long as it can rather than a simple SETNX which
+# would result in the master roll being passed around frequently.
+#
+# Locking Scheme: Each resque_admin-scheduler process attempts to get the master lock
+# via SETNX.  Once obtained, it sets the expiration for 3 minutes
+# (configurable).  The master process continually updates the timeout on the
+# lock key to be 3 minutes in the future in it's loop(s) (see `run`) and when
+# jobs come out of rufus-scheduler (see `load_schedule_job`).  That ensures
+# that a minimum of 3 minutes must pass since the last queuing operation before
+# a new master is chosen.  If, for whatever reason, the master fails to update
+# the expiration for 3 minutes, the key expires and the lock is up for grabs.
+# If miraculously the original master comes back to life, it will realize it is
+# no longer the master and stop processing jobs.
+#
+# The clocks on the scheduler machines can then be up to 3 minutes off from
+# each other without the risk of queueing the same scheduled job twice during a
+# master change.  The catch is, in the event of a master change, no scheduled
+# jobs will be queued during those 3 minutes.  So, there is a trade off: the
+# higher the timeout, the less likely scheduled jobs will be fired twice but
+# greater chances of missing scheduled jobs.  The lower the timeout, less
+# likely jobs will be missed, greater the chances of jobs firing twice.  If you
+# don't care about jobs firing twice or are certain your machines' clocks are
+# well in sync, a lower timeout is preferable.  One thing to keep in mind: this
+# only effects *scheduled* jobs - delayed jobs will never be lost or skipped
+# since eventually a master will come online and it will process everything
+# that is ready (no matter how old it is).  Scheduled jobs work like cron - if
+# you stop cron, no jobs fire while it's stopped and it doesn't fire jobs that
+# were missed when it starts up again.
+
+require_relative 'lock'
+
+module ResqueAdmin
+  module Scheduler
+    module Locking
+      def master_lock
+        @master_lock ||= build_master_lock
+      end
+
+      def supports_lua?
+        redis_master_version >= 2.5
+      end
+
+      def master?
+        master_lock.acquire! || master_lock.locked?
+      end
+
+      def release_master_lock!
+        warn "#{self}\#release_master_lock! is deprecated because it does " \
+             "not respect lock ownership. Use #{self}\#release_master_lock " \
+             "instead (at #{caller.first}"
+
+        master_lock.release!
+      end
+
+      def release_master_lock
+        master_lock.release
+      rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError
+        @master_lock = nil
+      end
+
+      private
+
+      def build_master_lock
+        if supports_lua?
+          ResqueAdmin::Scheduler::Lock::Resilient.new(master_lock_key)
+        else
+          ResqueAdmin::Scheduler::Lock::Basic.new(master_lock_key)
+        end
+      end
+
+      def master_lock_key
+        lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
+        lock_prefix += ':' if lock_prefix != ''
+        "#{ResqueAdmin.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
+      end
+
+      def redis_master_version
+        ResqueAdmin.redis.info['redis_version'].to_f
+      end
+    end
+  end
+end