resque-scheduler 2.2.0 → 4.10.2

data/lib/resque/scheduler/env.rb

@@ -0,0 +1,85 @@
+# vim:fileencoding=utf-8
+
+require 'English' # $PROCESS_ID
+
+module Resque
+  module Scheduler
+    class Env
+      def initialize(options)
+        @options = options
+        @pidfile_path = nil
+      end
+
+      def setup
+        require 'resque'
+        require 'resque/scheduler'
+
+        setup_backgrounding
+        setup_pid_file
+        setup_scheduler_configuration
+      end
+
+      def cleanup
+        cleanup_pid_file
+      end
+
+      private
+
+      attr_reader :options, :pidfile_path
+
+      def setup_backgrounding
+        return unless options[:background]
+
+        # Need to set this here for conditional Process.daemon redirect of
+        # stderr/stdout to /dev/null
+        Resque::Scheduler.quiet = if options.key?(:quiet)
+                                    !!options[:quiet]
+                                  else
+                                    true
+                                  end
+
+        Process.daemon(true, !Resque::Scheduler.quiet)
+        Resque.redis.reconnect
+      end
+
+      def setup_pid_file
+        return unless options[:pidfile]
+
+        @pidfile_path = File.expand_path(options[:pidfile])
+
+        File.open(pidfile_path, 'w') do |f|
+          f.puts $PROCESS_ID
+        end
+
+        at_exit { cleanup_pid_file }
+      end
+
+      def setup_scheduler_configuration
+        Resque::Scheduler.configure do |c|
+          c.app_name = options[:app_name] if options.key?(:app_name)
+
+          c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
+
+          c.env = options[:env] if options.key?(:env)
+
+          c.logfile = options[:logfile] if options.key?(:logfile)
+
+          c.logformat = options[:logformat] if options.key?(:logformat)
+
+          if (psleep = options[:poll_sleep_amount]) && !psleep.nil?
+            c.poll_sleep_amount = Float(psleep)
+          end
+
+          c.verbose = !!options[:verbose] if options.key?(:verbose)
+        end
+      end
+
+      def cleanup_pid_file
+        return unless pidfile_path
+
+        File.delete(pidfile_path) if File.exist?(pidfile_path)
+        @pidfile_path = nil
+      end
+    end
+  end
+end

data/lib/resque/scheduler/extension.rb

@@ -0,0 +1,13 @@
+# vim:fileencoding=utf-8
+
+require_relative 'scheduling_extensions'
+require_relative 'delaying_extensions'
+
+module Resque
+  module Scheduler
+    module Extension
+      include SchedulingExtensions
+      include DelayingExtensions
+    end
+  end
+end

data/lib/resque/scheduler/failure_handler.rb

@@ -0,0 +1,11 @@
+module Resque
+  module Scheduler
+    class FailureHandler
+      def self.on_enqueue_failure(_, e)
+        Resque::Scheduler.log_error(
+          "#{e.class.name}: #{e.message} #{e.backtrace.inspect}"
+        )
+      end
+    end
+  end
+end

data/lib/resque/scheduler/lock/base.rb

@@ -1,5 +1,7 @@
+# vim:fileencoding=utf-8
+
 module Resque
-  class Scheduler
+  module Scheduler
     module Lock
       class Base
         attr_reader :key
@@ -31,7 +33,12 @@ module Resque
           Resque.redis.del(key) == 1
         end
 
-      private
+        # Releases the lock iff we own it
+        def release
+          locked? && release!
+        end
+
+        private
 
         # Extends the lock by `timeout` seconds.
         def extend_lock!
@@ -40,7 +47,9 @@ module Resque
 
         def hostname
           local_hostname = Socket.gethostname
-          Socket.gethostbyname(local_hostname).first rescue local_hostname
+          Addrinfo.getaddrinfo(local_hostname, 'http').first.getnameinfo.first
+        rescue
+          local_hostname
         end
 
         def process_id

data/lib/resque/scheduler/lock/basic.rb

@@ -1,7 +1,8 @@
-require 'resque/scheduler/lock/base'
+# vim:fileencoding=utf-8
+require_relative 'base'
 
 module Resque
-  class Scheduler
+  module Scheduler
     module Lock
       class Basic < Base
         def acquire!
@@ -15,9 +16,7 @@ module Resque
           if Resque.redis.get(key) == value
             extend_lock!
 
-            if Resque.redis.get(key) == value
-              return true
-            end
+            return true if Resque.redis.get(key) == value
           end
 
           false

data/lib/resque/scheduler/lock/resilient.rb

@@ -1,67 +1,76 @@
-require 'resque/scheduler/lock/base'
+# vim:fileencoding=utf-8
+require_relative 'base'
 
 module Resque
-  class Scheduler
+  module Scheduler
     module Lock
       class Resilient < Base
         def acquire!
-          Resque.redis.evalsha(
-            acquire_sha,
-            :keys => [key],
-            :argv => [value]
-          ).to_i == 1
+          evalsha(:acquire, [key], [value]).to_i == 1
         end
 
         def locked?
-          Resque.redis.evalsha(
-            locked_sha,
-            :keys => [key],
-            :argv => [value]
-          ).to_i == 1
+          evalsha(:locked, [key], [value]).to_i == 1
         end
 
-      private
+        def timeout=(seconds)
+          if locked?
+            @timeout = seconds
+            @locked_sha = nil
+            @acquire_sha = nil
+          end
+          @timeout
+        end
+
+        private
+
+        def evalsha(script, keys, argv, refresh: false)
+          sha_method_name = "#{script}_sha"
+          Resque.redis.evalsha(
+            send(sha_method_name, refresh),
+            keys: keys,
+            argv: argv
+          )
+        rescue Redis::CommandError => e
+          if e.message =~ /NOSCRIPT/
+            refresh = true
+            retry
+          end
+          raise
+        end
 
         def locked_sha(refresh = false)
           @locked_sha = nil if refresh
 
-          @locked_sha ||= begin
-            Resque.redis.script(
-              :load,
-              <<-EOF
-if redis.call('GET', KEYS[1]) == ARGV[1]
-then
-  redis.call('EXPIRE', KEYS[1], #{timeout})
+          @locked_sha ||=
+            Resque.data_store.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+              if redis.call('GET', KEYS[1]) == ARGV[1]
+              then
+                redis.call('EXPIRE', KEYS[1], #{timeout})
 
-  if redis.call('GET', KEYS[1]) == ARGV[1]
-  then
-    return 1
-  end
-end
+                if redis.call('GET', KEYS[1]) == ARGV[1]
+                then
+                  return 1
+                end
+              end
 
-return 0
-EOF
-            )
-          end
+              return 0
+            EOF
         end
 
         def acquire_sha(refresh = false)
           @acquire_sha = nil if refresh
 
-          @acquire_sha ||= begin
-            Resque.redis.script(
-              :load,
-              <<-EOF
-if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
-then
-  redis.call('EXPIRE', KEYS[1], #{timeout})
-  return 1
-else
-  return 0
-end
-EOF
-            )
-          end
+          @acquire_sha ||=
+            Resque.data_store.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+              if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
+              then
+                redis.call('EXPIRE', KEYS[1], #{timeout})
+                return 1
+              else
+                return 0
+              end
+            EOF
         end
       end
     end

data/lib/resque/scheduler/lock.rb

@@ -1,3 +1,4 @@
-%w[base basic resilient].each do |file|
+# vim:fileencoding=utf-8
+%w(base basic resilient).each do |file|
   require "resque/scheduler/lock/#{file}"
 end

data/lib/resque/scheduler/locking.rb

@@ -0,0 +1,104 @@
+# vim:fileencoding=utf-8
+
+# ### Locking the scheduler process
+#
+# There are two places in resque-scheduler that need to be synchronized in order
+# to be able to run redundant scheduler processes while ensuring jobs don't get
+# queued multiple times when the master process changes.
+#
+# 1) Processing the delayed queues (jobs that are created from
+# enqueue_at/enqueue_in, etc) 2) Processing the scheduled (cron-like) jobs from
+# rufus-scheduler
+#
+# Protecting the delayed queues (#1) is relatively easy.  A simple SETNX in
+# redis would suffice.  However, protecting the scheduled jobs is trickier
+# because the clocks on machines could be slightly off or actual firing times
+# could vary slightly due to load.  If scheduler A's clock is slightly ahead of
+# scheduler B's clock (since they are on different machines), when scheduler A
+# dies, we need to ensure that scheduler B doesn't queue jobs that A already
+# queued before it's death. (This all assumes that it is better to miss a few
+# scheduled jobs than it is to run them multiple times for the same iteration.)
+#
+# To avoid queuing multiple jobs in the case of master fail-over, the master
+# should remain the master as long as it can rather than a simple SETNX which
+# would result in the master roll being passed around frequently.
+#
+# Locking Scheme: Each resque-scheduler process attempts to get the master lock
+# via SETNX.  Once obtained, it sets the expiration for 3 minutes
+# (configurable).  The master process continually updates the timeout on the
+# lock key to be 3 minutes in the future in it's loop(s) (see `run`) and when
+# jobs come out of rufus-scheduler (see `load_schedule_job`).  That ensures
+# that a minimum of 3 minutes must pass since the last queuing operation before
+# a new master is chosen.  If, for whatever reason, the master fails to update
+# the expiration for 3 minutes, the key expires and the lock is up for grabs.
+# If miraculously the original master comes back to life, it will realize it is
+# no longer the master and stop processing jobs.
+#
+# The clocks on the scheduler machines can then be up to 3 minutes off from
+# each other without the risk of queueing the same scheduled job twice during a
+# master change.  The catch is, in the event of a master change, no scheduled
+# jobs will be queued during those 3 minutes.  So, there is a trade off: the
+# higher the timeout, the less likely scheduled jobs will be fired twice but
+# greater chances of missing scheduled jobs.  The lower the timeout, less
+# likely jobs will be missed, greater the chances of jobs firing twice.  If you
+# don't care about jobs firing twice or are certain your machines' clocks are
+# well in sync, a lower timeout is preferable.  One thing to keep in mind: this
+# only effects *scheduled* jobs - delayed jobs will never be lost or skipped
+# since eventually a master will come online and it will process everything
+# that is ready (no matter how old it is).  Scheduled jobs work like cron - if
+# you stop cron, no jobs fire while it's stopped and it doesn't fire jobs that
+# were missed when it starts up again.
+
+require_relative 'lock'
+
+module Resque
+  module Scheduler
+    module Locking
+      def master_lock
+        @master_lock ||= build_master_lock
+      end
+
+      def supports_lua?
+        redis_master_version >= 2.5
+      end
+
+      def master?
+        master_lock.acquire! || master_lock.locked?
+      end
+
+      def release_master_lock!
+        warn "#{self}\#release_master_lock! is deprecated because it does " \
+             "not respect lock ownership. Use #{self}\#release_master_lock " \
+             "instead (at #{caller.first}"
+
+        master_lock.release!
+      end
+
+      def release_master_lock
+        master_lock.release
+      rescue *INTERMITTENT_ERRORS
+        @master_lock = nil
+      end
+
+      private
+
+      def build_master_lock
+        if supports_lua?
+          Resque::Scheduler::Lock::Resilient.new(master_lock_key)
+        else
+          Resque::Scheduler::Lock::Basic.new(master_lock_key)
+        end
+      end
+
+      def master_lock_key
+        lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
+        lock_prefix += ':' if lock_prefix != ''
+        "#{Resque.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
+      end
+
+      def redis_master_version
+        Resque.data_store.redis.info['redis_version'].to_f
+      end
+    end
+  end
+end

data/lib/resque/scheduler/logger_builder.rb

@@ -0,0 +1,83 @@
+# vim:fileencoding=utf-8
+
+require 'mono_logger'
+
+module Resque
+  module Scheduler
+    # Just builds a logger, with specified verbosity and destination.
+    # The simplest example:
+    #
+    #   Resque::Scheduler::LoggerBuilder.new.build
+    class LoggerBuilder
+      # Initializes new instance of the builder
+      #
+      # Pass :opts Hash with
+      #   - :quiet if logger needs to be silent for all levels. Default - false
+      #   - :verbose if there is a need in debug messages. Default - false
+      #   - :log_dev to output logs into a desired file. Default - STDOUT
+      #   - :format log format, either 'text', 'json' or 'logfmt'. Default - 'text'
+      #
+      # Example:
+      #
+      #   LoggerBuilder.new(
+      #     :quiet => false, :verbose => true, :log_dev => 'log/scheduler.log'
+      #   )
+      def initialize(opts = {})
+        @quiet = !!opts[:quiet]
+        @verbose = !!opts[:verbose]
+        @log_dev = opts[:log_dev] || $stdout
+        @format = opts[:format] || 'text'
+      end
+
+      # Returns an instance of MonoLogger
+      def build
+        logger = MonoLogger.new(@log_dev)
+        logger.progname = 'resque-scheduler'.freeze
+        logger.level = level
+        logger.formatter = send(:"#{@format}_formatter")
+        logger
+      end
+
+      private
+
+      def level
+        if @verbose && !@quiet
+          MonoLogger::DEBUG
+        elsif !@quiet
+          MonoLogger::INFO
+        else
+          MonoLogger::FATAL
+        end
+      end
+
+      def text_formatter
+        proc do |severity, datetime, progname, msg|
+          "#{progname}: [#{severity}] #{datetime.iso8601}: #{msg}\n"
+        end
+      end
+
+      def json_formatter
+        proc do |severity, datetime, progname, msg|
+          require 'json'
+          log_data = {
+            name: progname,
+            progname: progname,
+            level: severity,
+            timestamp: datetime.iso8601,
+            msg: msg
+          }
+          JSON.dump(log_data) + "\n"
+        end
+      end
+
+      def logfmt_formatter
+        proc do |severity, datetime, progname, msg|
+          "timestamp=\"#{datetime.iso8601}\" " \
+          "level=\"#{severity}\" " \
+          "progname=\"#{progname}\" " \
+          "msg=\"#{msg}\"\n"
+        end
+      end
+    end
+  end
+end

data/lib/resque/scheduler/plugin.rb

@@ -0,0 +1,31 @@
+# vim:fileencoding=utf-8
+
+module Resque
+  module Scheduler
+    module Plugin
+      def self.hooks(job, pattern)
+        job.methods.grep(/^#{pattern}/).sort
+      end
+
+      def self.run_hooks(job, pattern, *args)
+        results = hooks(job, pattern).map do |hook|
+          job.send(hook, *args)
+        end
+
+        results.all? { |result| result != false }
+      end
+
+      def self.run_before_delayed_enqueue_hooks(klass, *args)
+        run_hooks(klass, 'before_delayed_enqueue', *args)
+      end
+
+      def self.run_before_schedule_hooks(klass, *args)
+        run_hooks(klass, 'before_schedule', *args)
+      end
+
+      def self.run_after_schedule_hooks(klass, *args)
+        run_hooks(klass, 'after_schedule', *args)
+      end
+    end
+  end
+end

data/lib/resque/scheduler/scheduling_extensions.rb

@@ -0,0 +1,142 @@
+# vim:fileencoding=utf-8
+
+module Resque
+  module Scheduler
+    module SchedulingExtensions
+      # Accepts a new schedule configuration of the form:
+      #
+      #   {
+      #     "MakeTea" => {
+      #       "every" => "1m" },
+      #     "some_name" => {
+      #       "cron"        => "5/* * * *",
+      #       "class"       => "DoSomeWork",
+      #       "args"        => "work on this string",
+      #       "description" => "this thing works it"s butter off" },
+      #     ...
+      #   }
+      #
+      # Hash keys can be anything and are used to describe and reference
+      # the scheduled job. If the "class" argument is missing, the key
+      # is used implicitly as "class" argument - in the "MakeTea" example,
+      # "MakeTea" is used both as job name and resque worker class.
+      #
+      # Any jobs that were in the old schedule, but are not
+      # present in the new schedule, will be removed.
+      #
+      # :cron can be any cron scheduling string
+      #
+      # :every can be used in lieu of :cron. see rufus-scheduler's 'every'
+      # usage for valid syntax. If :cron is present it will take precedence
+      # over :every.
+      #
+      # :class must be a resque worker class. If it is missing, the job name
+      # (hash key) will be used as :class.
+      #
+      # :args can be any yaml which will be converted to a ruby literal and
+      # passed in a params. (optional)
+      #
+      # :rails_env is the list of envs where the job gets loaded. Envs are
+      # comma separated (optional)
+      #
+      # :description is just that, a description of the job (optional). If
+      # params is an array, each element in the array is passed as a separate
+      # param, otherwise params is passed in as the only parameter to
+      # perform.
+      def schedule=(schedule_hash)
+        @non_persistent_schedules = nil
+        prepared_schedules = prepare_schedules(schedule_hash)
+
+        prepared_schedules.each do |schedule, config|
+          set_schedule(schedule, config, false)
+        end
+
+        # ensure only return the successfully saved data!
+        reload_schedule!
+      end
+
+      # Returns the schedule hash
+      def schedule
+        @schedule ||= all_schedules
+        @schedule || {}
+      end
+
+      # reloads the schedule from redis and memory
+      def reload_schedule!
+        @schedule = all_schedules
+      end
+
+      # gets the schedules as it exists in redis
+      def all_schedules
+        non_persistent_schedules.merge(persistent_schedules)
+      end
+
+      # Create or update a schedule with the provided name and configuration.
+      #
+      # Note: values for class and custom_job_class need to be strings,
+      # not constants.
+      #
+      #    Resque.set_schedule('some_job', {:class => 'SomeJob',
+      #                                     :every => '15mins',
+      #                                     :queue => 'high',
+      #                                     :args => '/tmp/poop'})
+      #
+      # Preventing a reload is optional and available to batch operations
+      def set_schedule(name, config, reload = true)
+        persist = config.delete(:persist) || config.delete('persist')
+
+        if persist
+          redis.hset(:persistent_schedules, name, encode(config))
+        else
+          non_persistent_schedules[name] = decode(encode(config))
+        end
+
+        redis.sadd(:schedules_changed, [name])
+        reload_schedule! if reload
+      end
+
+      # retrive the schedule configuration for the given name
+      def fetch_schedule(name)
+        schedule[name]
+      end
+
+      # remove a given schedule by name
+      # Preventing a reload is optional and available to batch operations
+      def remove_schedule(name, reload = true)
+        non_persistent_schedules.delete(name)
+        redis.hdel(:persistent_schedules, name)
+        redis.sadd(:schedules_changed, [name])
+
+        reload_schedule! if reload
+      end
+
+      private
+
+      # we store our non-persistent schedules in this hash
+      def non_persistent_schedules
+        @non_persistent_schedules ||= {}
+      end
+
+      # reads the persistent schedules from redis
+      def persistent_schedules
+        redis.hgetall(:persistent_schedules).tap do |h|
+          h.each do |name, config|
+            h[name] = decode(config)
+          end
+        end
+      end
+
+      def prepare_schedules(schedule_hash)
+        prepared_hash = {}
+        schedule_hash.each do |name, job_spec|
+          job_spec = job_spec.dup
+          unless job_spec.key?('class') || job_spec.key?(:class)
+            job_spec['class'] = name
+          end
+          prepared_hash[name] = job_spec
+        end
+        prepared_hash
+      end
+    end
+  end
+end