resque-scheduler 2.5.1 → 4.10.2

data/lib/resque/scheduler.rb

@@ -1,103 +1,43 @@
+# vim:fileencoding=utf-8
+
+require 'redis/errors'
 require 'rufus/scheduler'
-require 'resque/scheduler_locking'
-require 'resque_scheduler/logger_builder'
+require_relative 'scheduler/configuration'
+require_relative 'scheduler/locking'
+require_relative 'scheduler/logger_builder'
+require_relative 'scheduler/signal_handling'
+require_relative 'scheduler/failure_handler'
 
 module Resque
-  class Scheduler
-    extend Resque::SchedulerLocking
-
-    class << self
-      # Allows for block-style configuration
-      def configure
-        yield self
-      end
-
-      attr_writer :signal_queue
-
-      def signal_queue
-        @signal_queue ||= []
-      end
-
-      # Used in `#load_schedule_job`
-      attr_writer :env
+  module Scheduler
+    autoload :Cli, 'resque/scheduler/cli'
+    autoload :Extension, 'resque/scheduler/extension'
+    autoload :Util, 'resque/scheduler/util'
+    autoload :VERSION, 'resque/scheduler/version'
+    INTERMITTENT_ERRORS = [
+      Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError, Redis::TimeoutError
+    ].freeze
 
-      def env
-        return @env if @env
-        @env ||= Rails.env if defined?(Rails)
-        @env ||= ENV['RAILS_ENV']
-        @env
-      end
-
-      # If true, logs more stuff...
-      attr_writer :verbose
-
-      def verbose
-        @verbose ||= !!ENV['VERBOSE']
-      end
+    private
 
-      # If set, produces no output
-      attr_writer :mute
+    extend Resque::Scheduler::Locking
+    extend Resque::Scheduler::Configuration
+    extend Resque::Scheduler::SignalHandling
 
-      def mute
-        @mute ||= !!ENV['MUTE']
-      end
-
-      # If set, will write messages to the file
-      attr_writer :logfile
-
-      def logfile
-        @logfile ||= ENV['LOGFILE']
-      end
-
-      # Sets whether to log in 'text' or 'json'
-      attr_writer :logformat
-
-      def logformat
-        @logformat ||= ENV['LOGFORMAT']
-      end
-
-      # If set, will try to update the schedule in the loop
-      attr_writer :dynamic
-
-      def dynamic
-        @dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
-      end
-
-      # If set, will append the app name to procline
-      attr_writer :app_name
-
-      def app_name
-        @app_name ||= ENV['APP_NAME']
-      end
-
-      # Amount of time in seconds to sleep between polls of the delayed
-      # queue.  Defaults to 5
-      attr_writer :poll_sleep_amount
-
-      def poll_sleep_amount
-        @poll_sleep_amount ||=
-          Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
-      end
+    public
 
+    class << self
       attr_writer :logger
 
-      def logger
-        @logger ||= ResqueScheduler::LoggerBuilder.new(
-          :mute => mute,
-          :verbose => verbose,
-          :log_dev => logfile,
-          :format => logformat
-        ).build
-      end
-
       # the Rufus::Scheduler jobs that are scheduled
-      def scheduled_jobs
-        @@scheduled_jobs
-      end
+      attr_reader :scheduled_jobs
+
+      # allow user to set an additional failure handler
+      attr_writer :failure_handler
 
       # Schedule all jobs and continually look for delayed jobs (never returns)
       def run
-        $0 = "resque-scheduler: Starting"
+        procline 'Starting'
 
         # trap signals
         register_signal_handlers
@@ -108,28 +48,32 @@ module Resque
         $stdout.sync = true
         $stderr.sync = true
 
-        # Load the schedule into rufus
-        # If dynamic is set, load that schedule otherwise use normal load
-        if dynamic
-          reload_schedule!
-        else
-          load_schedule!
-        end
+        was_master = nil
 
         begin
           @th = Thread.current
 
           # Now start the scheduling part of the loop.
           loop do
-            if is_master?
-              begin
+            begin
+              # Check on changes to master/child
+              @am_master = master?
+              if am_master != was_master
+                procline am_master ? 'Master scheduler' : 'Child scheduler'
+
+                # Load schedule because changed
+                reload_schedule!
+              end
+
+              if am_master
                 handle_delayed_items
                 update_schedule if dynamic
-              rescue Errno::EAGAIN, Errno::ECONNRESET => e
-                log! e.message
               end
+              was_master = am_master
+            rescue *INTERMITTENT_ERRORS => e
+              log! e.message
+              release_master_lock
             end
-            handle_signals
             poll_sleep
           end
 
@@ -138,33 +82,11 @@ module Resque
         end
       end
 
-      # For all signals, set the shutdown flag and wait for current
-      # poll/enqueing to finish (should be almost instant).  In the
-      # case of sleeping, exit immediately.
-      def register_signal_handlers
-        %w(INT TERM USR1 USR2 QUIT).each do |sig|
-          trap(sig) { signal_queue << sig }
-        end
-      end
-
-      def handle_signals
-        loop do
-          sig = signal_queue.shift
-          break unless sig
-          log! "Got #{sig} signal"
-          case sig
-          when 'INT', 'TERM', 'QUIT' then shutdown
-          when 'USR1' then print_schedule
-          when 'USR2' then reload_schedule!
-          end
-        end
-      end
-
       def print_schedule
         if rufus_scheduler
           log! "Scheduling Info\tLast Run"
-          scheduler_jobs = rufus_scheduler.all_jobs
-          scheduler_jobs.each do |k, v|
+          scheduler_jobs = rufus_scheduler.jobs
+          scheduler_jobs.each do |_k, v|
             log! "#{v.t}\t#{v.last}\t"
           end
         end
@@ -173,20 +95,20 @@ module Resque
       # Pulls the schedule from Resque.schedule and loads it into the
       # rufus scheduler instance
       def load_schedule!
-        procline "Loading Schedule"
+        procline 'Loading Schedule'
 
         # Need to load the schedule from redis for the first time if dynamic
         Resque.reload_schedule! if dynamic
 
-        log! "Schedule empty! Set Resque.schedule" if Resque.schedule.empty?
+        log! 'Schedule empty! Set Resque.schedule' if Resque.schedule.empty?
 
-        @@scheduled_jobs = {}
+        @scheduled_jobs = {}
 
         Resque.schedule.each do |name, config|
           load_schedule_job(name, config)
         end
-        Resque.redis.del(:schedules_changed)
-        procline "Schedules Loaded"
+        Resque.redis.del(:schedules_changed) if am_master && dynamic
+        procline 'Schedules Loaded'
       end
 
       # modify interval type value to value with options if options available
@@ -195,17 +117,20 @@ module Resque
         if args.is_a?(::Array)
           return args.first if args.size > 2 || !args.last.is_a?(::Hash)
           # symbolize keys of hash for options
-          args[1] = args[1].inject({}) do |m, i|
+          args[2] = args[1].reduce({}) do |m, i|
             key, value = i
-            m[(key.to_sym rescue key) || key] = value
+            m[(key.respond_to?(:to_sym) ? key.to_sym : key) || key] = value
             m
           end
+
+          args[2][:job] = true
+          args[1] = nil
         end
         args
       end
 
-      # Loads a job schedule into the Rufus::Scheduler and stores it in
-      # @@scheduled_jobs
+      # Loads a job schedule into the Rufus::Scheduler and stores it
+      # in @scheduled_jobs
       def load_schedule_job(name, config)
         # If `rails_env` or `env` is set in the config, load jobs only if they
         # are meant to be loaded in `Resque::Scheduler.env`.  If `rails_env` or
@@ -217,33 +142,33 @@ module Resque
         if configured_env.nil? || env_matches?(configured_env)
           log! "Scheduling #{name} "
           interval_defined = false
-          interval_types = %w{cron every}
+          interval_types = %w(cron every)
           interval_types.each do |interval_type|
-            if !config[interval_type].nil? && config[interval_type].length > 0
-              args = optionizate_interval_value(config[interval_type])
-              @@scheduled_jobs[name] = rufus_scheduler.send(interval_type, *args) do
-                if is_master?
-                  log! "queueing #{config['class']} (#{name})"
-                  handle_errors { enqueue_from_config(config) }
-                end
-              end
-              interval_defined = true
-              break
+            next unless !config[interval_type].nil? && !config[interval_type].empty?
+            args = optionizate_interval_value(config[interval_type])
+            args = [args, nil, job: true] if args.is_a?(::String)
+
+            job = rufus_scheduler.send(interval_type, *args) do
+              enqueue_recurring(name, config)
             end
+            @scheduled_jobs[name] = job
+            interval_defined = true
+            break
           end
           unless interval_defined
-            log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
+            log! "no #{interval_types.join(' / ')} found for " \
+                 "#{config['class']} (#{name}) - skipping"
           end
         else
-          log "Skipping schedule of #{name} because configured " <<
-              "env #{configured_env.inspect} does not match current " <<
+          log "Skipping schedule of #{name} because configured " \
+              "env #{configured_env.inspect} does not match current " \
               "env #{env.inspect}"
         end
       end
 
       # Returns true if the given schedule config hash matches the current env
       def rails_env_matches?(config)
-        warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' <<
+        warn '`Resque::Scheduler.rails_env_matches?` is deprecated. ' \
              'Please use `Resque::Scheduler.env_matches?` instead.'
         config['rails_env'] && env &&
           config['rails_env'].split(/[\s,]+/).include?(env)
@@ -257,29 +182,110 @@ module Resque
 
       # Handles queueing delayed items
       # at_time - Time to start scheduling items (default: now).
-      def handle_delayed_items(at_time=nil)
-        if timestamp = Resque.next_delayed_timestamp(at_time)
-          procline "Processing Delayed Items"
-          while !timestamp.nil?
+      def handle_delayed_items(at_time = nil)
+        timestamp = Resque.next_delayed_timestamp(at_time)
+        if timestamp
+          procline 'Processing Delayed Items'
+          until timestamp.nil?
             enqueue_delayed_items_for_timestamp(timestamp)
             timestamp = Resque.next_delayed_timestamp(at_time)
           end
         end
       end
 
+      def enqueue_next_item(timestamp)
+        item = Resque.next_item_for_timestamp(timestamp)
+
+        if item
+          log "queuing #{item['class']} [delayed]"
+          enqueue(item)
+        end
+
+        item
+      end
+
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
-        item = nil
-        begin
+        count = 0
+        batch_size = delayed_requeue_batch_size
+        actual_batch_size = nil
+
+        log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
+
+        loop do
           handle_shutdown do
             # Continually check that it is still the master
-            if is_master? && item = Resque.next_item_for_timestamp(timestamp)
-              log "queuing #{item['class']} [delayed]"
-              handle_errors { enqueue_from_config(item) }
+            if am_master
+              actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
+                                                                       batch_size)
             end
           end
-        # continue processing until there are no more ready items in this timestamp
-        end while !item.nil?
+
+          count += actual_batch_size
+          log "queued #{count} jobs" if actual_batch_size != -1
+
+          # continue processing until there are no more items in this
+          # timestamp. If we don't have a full batch, this is the last one.
+          # This also breaks us in the event of a redis transaction failure
+          # i.e. enqueue_items_in_batch_for_timestamp returned -1
+          break if actual_batch_size < batch_size
+        end
+
+        log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
+      end
+
+      def timestamp_key(timestamp)
+        "delayed:#{timestamp.to_i}"
+      end
+
+      def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
+        timestamp_bucket_key = timestamp_key(timestamp)
+
+        encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
+
+        # Watch is used to ensure that the timestamp bucket we are operating on
+        # is not altered by any other clients between the watch call and when we call exec
+        # (to execute the multi block). We should error catch on the redis.exec return value
+        # as that will indicate if the entire transaction was aborted or not. Though we should
+        # be safe as our ltrim is inside the multi block and therefore also would have been
+        # aborted. So nothing would have been queued, but also nothing lost from the bucket.
+        watch_result = Resque.redis.watch(timestamp_bucket_key) do
+          Resque.redis.multi do |pipeline|
+            encoded_jobs_to_requeue.each do |encoded_job|
+              pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
+
+              decoded_job = Resque.decode(encoded_job)
+              enqueue(decoded_job)
+            end
+
+            pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
+          end
+        end
+
+        # Did the multi block successfully remove from this timestamp and enqueue the jobs?
+        success = !watch_result.nil?
+
+        # If this was the last batch in this timestamp bucket, clean up
+        if success && encoded_jobs_to_requeue.count < batch_size
+          Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
+        end
+
+        unless success
+          # Our batched transaction failed in Redis due to the timestamp_bucket_key value
+          # being modified while we built our multi block. We return -1 to ensure we break
+          # out of the loop iterating on this timestamp so it can be re-processed via the
+          # loop in handle_delayed_items.
+          return -1
+        end
+
+        # will return 0 if none were left to batch
+        encoded_jobs_to_requeue.count
+      end
+
+      def enqueue(config)
+        enqueue_from_config(config)
+      rescue => e
+        Resque::Scheduler.failure_handler.on_enqueue_failure(config, e)
       end
 
       def handle_shutdown
@@ -288,43 +294,50 @@ module Resque
         exit if @shutdown
       end
 
-      def handle_errors
-        begin
-          yield
-        rescue Exception => e
-          log_error "#{e.class.name}: #{e.message}"
-        end
-      end
-
       # Enqueues a job based on a config hash
       def enqueue_from_config(job_config)
         args = job_config['args'] || job_config[:args]
 
         klass_name = job_config['class'] || job_config[:class]
-        klass = ResqueScheduler::Util.constantize(klass_name) rescue klass_name
+        begin
+          klass = Resque::Scheduler::Util.constantize(klass_name)
+        rescue NameError
+          klass = klass_name
+        end
 
         params = args.is_a?(Hash) ? [args] : Array(args)
-        queue = job_config['queue'] || job_config[:queue] || Resque.queue_from_class(klass)
-        # Support custom job classes like those that inherit from Resque::JobWithStatus (resque-status)
-        if (job_klass = job_config['custom_job_class']) && (job_klass != 'Resque::Job')
-          # The custom job class API must offer a static "scheduled" method. If the custom
-          # job class can not be constantized (via a requeue call from the web perhaps), fall
-          # back to enqueing normally via Resque::Job.create.
+        queue = job_config['queue'] ||
+                job_config[:queue] ||
+                Resque.queue_from_class(klass)
+        # Support custom job classes like those that inherit from
+        # Resque::JobWithStatus (resque-status)
+        job_klass = job_config['custom_job_class']
+        if job_klass && job_klass != 'Resque::Job'
+          # The custom job class API must offer a static "scheduled" method. If
+          # the custom job class can not be constantized (via a requeue call
+          # from the web perhaps), fall back to enqueuing normally via
+          # Resque::Job.create.
           begin
-            ResqueScheduler::Util.constantize(job_klass).scheduled(queue, klass_name, *params)
+            Resque::Scheduler::Util.constantize(job_klass).scheduled(
+              queue, klass_name, *params
+            )
           rescue NameError
-            # Note that the custom job class (job_config['custom_job_class']) is the one enqueued
+            # Note that the custom job class (job_config['custom_job_class'])
+            # is the one enqueued
             Resque::Job.create(queue, job_klass, *params)
           end
         else
-          # hack to avoid havoc for people shoving stuff into queues
+          # Hack to avoid havoc for people shoving stuff into queues
           # for non-existent classes (for example: running scheduler in
-          # one app that schedules for another
+          # one app that schedules for another.
           if Class === klass
-            ResqueScheduler::Plugin.run_before_delayed_enqueue_hooks(klass, *params)
+            Resque::Scheduler::Plugin.run_before_delayed_enqueue_hooks(
+              klass, *params
+            )
 
-            # If the class is a custom job class, call self#scheduled on it. This allows you to do things like
-            # Resque.enqueue_at(timestamp, CustomJobClass). Otherwise, pass off to Resque.
+            # If the class is a custom job class, call self#scheduled on it.
+            # This allows you to do things like Resque.enqueue_at(timestamp,
+            # CustomJobClass). Otherwise, pass off to Resque.
             if klass.respond_to?(:scheduled)
               klass.scheduled(queue, klass_name, *params)
             else
@@ -339,7 +352,7 @@ module Resque
       end
 
       def rufus_scheduler
-        @rufus_scheduler ||= Rufus::Scheduler.start_new
+        @rufus_scheduler ||= Rufus::Scheduler.new
       end
 
       # Stops old rufus scheduler and creates a new one.  Returns the new
@@ -347,21 +360,23 @@ module Resque
       def clear_schedule!
         rufus_scheduler.stop
         @rufus_scheduler = nil
-        @@scheduled_jobs = {}
+        @scheduled_jobs = {}
         rufus_scheduler
       end
 
       def reload_schedule!
-        procline "Reloading Schedule"
+        procline 'Reloading Schedule'
         clear_schedule!
         load_schedule!
       end
 
       def update_schedule
         if Resque.redis.scard(:schedules_changed) > 0
-          procline "Updating schedule"
-          Resque.reload_schedule!
-          while schedule_name = Resque.redis.spop(:schedules_changed)
+          procline 'Updating schedule'
+          loop do
+            schedule_name = Resque.redis.spop(:schedules_changed)
+            break unless schedule_name
+            Resque.reload_schedule!
             if Resque.schedule.keys.include?(schedule_name)
               unschedule_job(schedule_name)
               load_schedule_job(schedule_name, Resque.schedule[schedule_name])
@@ -369,7 +384,7 @@ module Resque
               unschedule_job(schedule_name)
             end
           end
-          procline "Schedules Loaded"
+          procline 'Schedules Loaded'
         end
       end
 
@@ -377,7 +392,7 @@ module Resque
         if scheduled_jobs[name]
           log "Removing schedule #{name}"
           scheduled_jobs[name].unschedule
-          @@scheduled_jobs.delete(name)
+          @scheduled_jobs.delete(name)
         end
       end
 
@@ -385,16 +400,7 @@ module Resque
       def poll_sleep
         handle_shutdown do
           begin
-            begin
-              @sleeping = true
-              sleep poll_sleep_amount
-              @sleeping = false
-            rescue Interrupt
-              if @shutdown
-                Resque.clean_schedules
-                release_master_lock!
-              end
-            end
+            poll_sleep_loop
           ensure
             @sleeping = false
           end
@@ -402,6 +408,46 @@ module Resque
         true
       end
 
+      def poll_sleep_loop
+        @sleeping = true
+        if poll_sleep_amount > 0
+          start = Time.now
+          loop do
+            elapsed_sleep = (Time.now - start)
+            remaining_sleep = poll_sleep_amount - elapsed_sleep
+            @do_break = false
+            if remaining_sleep <= 0
+              @do_break = true
+            else
+              @do_break = handle_signals_with_operation do
+                sleep(remaining_sleep)
+              end
+            end
+            break if @do_break
+          end
+        else
+          handle_signals_with_operation
+        end
+      end
+
+      def handle_signals_with_operation
+        yield if block_given?
+        handle_signals
+        false
+      rescue Interrupt
+        before_shutdown if @shutdown
+        true
+      end
+
+      def stop_rufus_scheduler
+        rufus_scheduler.shutdown(:wait)
+      end
+
+      def before_shutdown
+        stop_rufus_scheduler
+        release_master_lock
+      end
+
       # Sets the shutdown flag, clean schedules and exits if sleeping
       def shutdown
         return if @shutdown
@@ -429,8 +475,29 @@ module Resque
         $0 = argv0
       end
 
+      def failure_handler
+        @failure_handler ||= Resque::Scheduler::FailureHandler
+      end
+
+      def logger
+        @logger ||= Resque::Scheduler::LoggerBuilder.new(
+          quiet: quiet,
+          verbose: verbose,
+          log_dev: logfile,
+          format: logformat
+        ).build
+      end
+
       private
 
+      def enqueue_recurring(name, config)
+        if am_master
+          log! "queueing #{config['class']} (#{name})"
+          enqueue(config)
+          Resque.last_enqueued_at(name, Time.now.to_s)
+        end
+      end
+
       def app_str
         app_name ? "[#{app_name}]" : ''
       end
@@ -444,7 +511,12 @@ module Resque
       end
 
       def internal_name
-        "resque-scheduler-#{ResqueScheduler::VERSION}"
+        "resque-scheduler-#{Resque::Scheduler::VERSION}"
+      end
+
+      def am_master
+        @am_master = master? unless defined?(@am_master)
+        @am_master
       end
     end
   end

data/lib/resque-scheduler.rb

@@ -1,2 +1,4 @@
 # vim:fileencoding=utf-8
-require File.expand_path('../resque_scheduler', __FILE__)
+require_relative 'resque/scheduler'
+
+Resque.extend Resque::Scheduler::Extension