resque-scheduler 2.3.1 → 2.4.0

data/lib/resque/scheduler/lock/resilient.rb

@@ -3,7 +3,7 @@ require 'resque/scheduler/lock/base'
 module Resque
   class Scheduler
     module Lock
-      class Resilient < Base
+      class Resilient < Base # rubocop:disable TrailingComma
         def acquire!
           Resque.redis.evalsha(
             acquire_sha,

data/lib/resque/scheduler_locking.rb

@@ -79,7 +79,9 @@ module Resque
     end
 
     def master_lock_key
-      "#{ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''}resque_scheduler_master_lock".to_sym
+      lock_prefix = ENV['RESQUE_SCHEDULER_MASTER_LOCK_PREFIX'] || ''
+      lock_prefix += ':' if lock_prefix != ''
+      "#{Resque.redis.namespace}:#{lock_prefix}resque_scheduler_master_lock"
     end
 
     def redis_master_version

data/lib/resque_scheduler.rb

@@ -6,6 +6,8 @@ require 'resque/scheduler'
 require 'resque_scheduler/plugin'
 
 module ResqueScheduler
+  autoload :Cli, 'resque_scheduler/cli'
+
   #
   # Accepts a new schedule configuration of the form:
   #
@@ -91,7 +93,7 @@ module ResqueScheduler
   def clean_schedules
     if redis.exists(:schedules)
       redis.hkeys(:schedules).each do |key|
-        remove_schedule(key)
+        remove_schedule(key) if !schedule_persisted?(key)
       end
     end
     @schedule = nil
@@ -109,9 +111,15 @@ module ResqueScheduler
   #                                     :args => '/tmp/poop'})
   def set_schedule(name, config)
     existing_config = get_schedule(name)
+    persist = config.delete(:persist) || config.delete('persist')
     unless existing_config && existing_config == config
-      redis.hset(:schedules, name, encode(config))
-      redis.sadd(:schedules_changed, name)
+      redis.pipelined do
+        redis.hset(:schedules, name, encode(config))
+        redis.sadd(:schedules_changed, name)
+        if persist
+          redis.sadd(:persisted_schedules, name)
+        end
+      end
     end
     config
   end
@@ -121,10 +129,17 @@ module ResqueScheduler
     decode(redis.hget(:schedules, name))
   end
 
+  def schedule_persisted?(name)
+    redis.sismember(:persisted_schedules, name)
+  end
+
   # remove a given schedule by name
   def remove_schedule(name)
-    redis.hdel(:schedules, name)
-    redis.sadd(:schedules_changed, name)
+    redis.pipelined do
+      redis.hdel(:schedules, name)
+      redis.srem(:persisted_schedules, name)
+      redis.sadd(:schedules_changed, name)
+    end
   end
 
   # This method is nearly identical to +enqueue+ only it also
@@ -272,6 +287,33 @@ module ResqueScheduler
     remove_delayed(klass, *args).times { Resque::Scheduler.enqueue_from_config(hash) }
   end
 
+  # Given a block, remove jobs that return true from a block
+  #
+  # This allows for removal of delayed jobs that have arguments matching certain criteria
+  def remove_delayed_selection
+    fail ArgumentError, "Please supply a block" unless block_given?
+
+    destroyed = 0
+    # There is no way to search Redis list entries for a partial match, so we query for all
+    # delayed job tasks and do our matching after decoding the payload data
+    jobs = Resque.redis.keys("delayed:*")
+    jobs.each do |job|
+      index = Resque.redis.llen(job) - 1
+      while index >= 0
+        payload = Resque.redis.lindex(job, index)
+        decoded_payload = decode(payload)
+        if yield(decoded_payload['args'])
+          removed = redis.lrem job, 0, payload
+          destroyed += removed
+          index -= removed
+        else
+          index -= 1
+        end
+      end
+    end
+    destroyed
+  end
+
   # Given a timestamp and job (klass + args) it removes all instances and
   # returns the count of jobs removed.
   #
@@ -306,39 +348,39 @@ module ResqueScheduler
 
   private
 
-    def job_to_hash(klass, args)
-      {:class => klass.to_s, :args => args, :queue => queue_from_class(klass)}
-    end
+  def job_to_hash(klass, args)
+    {:class => klass.to_s, :args => args, :queue => queue_from_class(klass)}
+  end
 
-    def job_to_hash_with_queue(queue, klass, args)
-      {:class => klass.to_s, :args => args, :queue => queue}
-    end
+  def job_to_hash_with_queue(queue, klass, args)
+    {:class => klass.to_s, :args => args, :queue => queue}
+  end
 
-    def clean_up_timestamp(key, timestamp)
-      # If the list is empty, remove it.
+  def clean_up_timestamp(key, timestamp)
+    # If the list is empty, remove it.
 
-      # Use a watch here to ensure nobody adds jobs to this delayed
-      # queue while we're removing it.
-      redis.watch key
-      if 0 == redis.llen(key).to_i
-        redis.multi do
-          redis.del key
-          redis.zrem :delayed_queue_schedule, timestamp.to_i
-        end
-      else
-        redis.unwatch
+    # Use a watch here to ensure nobody adds jobs to this delayed
+    # queue while we're removing it.
+    redis.watch key
+    if 0 == redis.llen(key).to_i
+      redis.multi do
+        redis.del key
+        redis.zrem :delayed_queue_schedule, timestamp.to_i
       end
+    else
+      redis.unwatch
     end
+  end
 
-    def prepare_schedule(schedule_hash)
-      prepared_hash = {}
-      schedule_hash.each do |name, job_spec|
-        job_spec = job_spec.dup
-        job_spec['class'] = name unless job_spec.key?('class') || job_spec.key?(:class)
-        prepared_hash[name] = job_spec
-      end
-      prepared_hash
+  def prepare_schedule(schedule_hash)
+    prepared_hash = {}
+    schedule_hash.each do |name, job_spec|
+      job_spec = job_spec.dup
+      job_spec['class'] = name unless job_spec.key?('class') || job_spec.key?(:class)
+      prepared_hash[name] = job_spec
     end
+    prepared_hash
+  end
 end
 
 Resque.extend ResqueScheduler

data/lib/resque_scheduler/cli.rb

@@ -0,0 +1,160 @@
+# vim:fileencoding=utf-8
+
+require 'optparse'
+
+module ResqueScheduler
+  class Cli
+    BANNER = <<-EOF.gsub(/ {6}/, '')
+      Usage: resque-scheduler [options]
+
+      Runs a resque scheduler process directly (rather than via rake).
+
+    EOF
+    OPTIONS = [
+      {
+        args: ['-n', '--app-name [APP_NAME]', 'Application name for procline'],
+        callback: ->(options) { ->(n) { options[:app_name] = n } }
+      },
+      {
+        args: ['-B', '--background', 'Run in the background [BACKGROUND]'],
+        callback: ->(options) { ->(b) { options[:background] = b  } }
+      },
+      {
+        args: ['-D', '--dynamic-schedule',
+               'Enable dynamic scheduling [DYNAMIC_SCHEDULE]'],
+        callback: ->(options) { ->(d) { options[:dynamic] = d } }
+      },
+      {
+        args: ['-E', '--environment [RAILS_ENV]', 'Environment name'],
+        callback: ->(options) { ->(e) { options[:env] = e } }
+      },
+      {
+        args: ['-I', '--initializer-path [INITIALIZER_PATH]',
+               'Path to optional initializer ruby file'],
+        callback: ->(options) { ->(i) { options[:initializer_path] = i } }
+      },
+      {
+        args: ['-i', '--interval [RESQUE_SCHEDULER_INTERVAL]',
+               'Interval for checking if a scheduled job must run'],
+        callback: ->(options) { ->(i) { options[:poll_sleep_amount] = i } }
+      },
+      {
+        args: ['-l', '--logfile [LOGFILE]', 'Log file name'],
+        callback: ->(options) { ->(l) { options[:logfile] = l } }
+      },
+      {
+        args: ['-F', '--logformat [LOGFORMAT]', 'Log output format'],
+        callback: ->(options) { ->(f) { options[:logformat] = f } }
+      },
+      {
+        args: ['-P', '--pidfile [PIDFILE]', 'PID file name'],
+        callback: ->(options) { ->(p) { options[:pidfile] = p } }
+      },
+      {
+        args: ['-q', '--quiet', 'Run with minimal output [QUIET] (or [MUTE])'],
+        callback: ->(options) { ->(q) { options[:mute] = q } }
+      },
+      {
+        args: ['-v', '--verbose', 'Run with verbose output [VERBOSE]'],
+        callback: ->(options) { ->(v) { options[:verbose] = v } }
+      }
+    ].freeze
+
+    def self.run!(argv = ARGV, env = ENV)
+      new(argv, env).run!
+    end
+
+    def initialize(argv = ARGV, env = ENV)
+      @argv = argv
+      @env = env
+    end
+
+    def run!
+      pre_run
+      run_forever
+    end
+
+    def pre_run
+      parse_options
+      pre_setup
+      setup_env
+    end
+
+    def parse_options
+      OptionParser.new do |opts|
+        opts.banner = BANNER
+        OPTIONS.each do |opt|
+          opts.on(*opt[:args], &(opt[:callback].call(options)))
+        end
+      end.parse!(argv.dup)
+    end
+
+    def pre_setup
+      if options[:initializer_path]
+        load options[:initializer_path].to_s.strip
+      else
+        false
+      end
+    end
+
+    def setup_env
+      require 'resque'
+      require 'resque/scheduler'
+
+      # Need to set this here for conditional Process.daemon redirect of
+      # stderr/stdout to /dev/null
+      Resque::Scheduler.mute = !!options[:mute]
+
+      if options[:background]
+        unless Process.respond_to?('daemon')
+          abort 'background option is set, which requires ruby >= 1.9'
+        end
+
+        Process.daemon(true, !Resque::Scheduler.mute)
+        Resque.redis.client.reconnect
+      end
+
+      File.open(options[:pidfile], 'w') do |f|
+        f.puts $PROCESS_ID
+      end if options[:pidfile]
+
+      Resque::Scheduler.configure do |c|
+        # These settings are somewhat redundant given the defaults present
+        # in the attr reader methods.  They are left here for clarity and
+        # to serve as an example of how to use `.configure`.
+
+        c.app_name = options[:app_name]
+        c.dynamic = !!options[:dynamic]
+        c.env = options[:env]
+        c.logfile = options[:logfile]
+        c.logformat = options[:logformat]
+        c.poll_sleep_amount = Float(options[:poll_sleep_amount] || '5')
+        c.verbose = !!options[:verbose]
+      end
+    end
+
+    def run_forever
+      Resque::Scheduler.run
+    end
+
+    private
+
+    attr_reader :argv, :env
+
+    def options
+      @options ||= {
+        app_name: env['APP_NAME'],
+        background: env['BACKGROUND'],
+        dynamic: env['DYNAMIC_SCHEDULE'],
+        env: env['RAILS_ENV'],
+        initializer_path: env['INITIALIZER_PATH'],
+        logfile: env['LOGFILE'],
+        logformat: env['LOGFORMAT'],
+        mute: env['MUTE'] || env['QUIET'],
+        pidfile: env['PIDFILE'],
+        poll_sleep_amount: env['RESQUE_SCHEDULER_INTERVAL'],
+        verbose: env['VERBOSE']
+      }
+    end
+  end
+end

data/lib/resque_scheduler/logger_builder.rb

@@ -1,3 +1,5 @@
+# vim:fileencoding=utf-8
+
 module ResqueScheduler
   # Just builds a logger, with specified verbosity and destination.
   # The simplest example:
@@ -10,23 +12,25 @@ module ResqueScheduler
     #   - :mute if logger needs to be silent for all levels. Default - false
     #   - :verbose if there is a need in debug messages. Default - false
     #   - :log_dev to output logs into a desired file. Default - STDOUT
+    #   - :format log format, either 'text' or 'json'. Default - 'text'
     #
     # Example:
     #
-    #   LoggerBuilder.new(:mute => false, :verbose => true, :log_dev => 'log/sheduler.log')
+    #   LoggerBuilder.new(
+    #     :mute => false, :verbose => true, :log_dev => 'log/scheduler.log'
+    #   )
     def initialize(opts={})
-      @muted   = !!opts[:mute]
+      @muted = !!opts[:mute]
       @verbose = !!opts[:verbose]
-      @log_dev = opts[:log_dev] || STDOUT
+      @log_dev = opts[:log_dev] || $stdout
+      @format = opts[:format] || 'text'
     end
 
     # Returns an instance of Logger
     def build
       logger = Logger.new(@log_dev)
       logger.level = level
-      logger.datetime_format = "%Y-%m-%d %H:%M:%S"
-      logger.formatter = formatter
-
+      logger.formatter = send(:"#{@format}_formatter")
       logger
     end
 
@@ -42,9 +46,24 @@ module ResqueScheduler
       end
     end
 
-    def formatter
+    def text_formatter
+      proc do |severity, datetime, progname, msg|
+        "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
+      end
+    end
+
+    def json_formatter
       proc do |severity, datetime, progname, msg|
-        "[#{severity}] #{datetime}: #{msg}\n"
+        require 'json'
+        JSON.dump(
+          {
+            :name => 'resque-scheduler',
+            :progname => progname,
+            :level => severity,
+            :timestamp => datetime.iso8601,
+            :msg => msg
+          }
+        ) + "\n"
       end
     end
   end

data/lib/resque_scheduler/plugin.rb

@@ -13,13 +13,16 @@ module ResqueScheduler
       results.all? { |result| result != false }
     end
 
-    def method_missing(method_name, *args, &block)
-      if method_name.to_s =~ /^run_(.*)_hooks$/
-        job = args.shift
-        run_hooks job, $1, *args
-      else
-        super
-      end
+    def run_before_delayed_enqueue_hooks(klass, *args)
+      run_hooks(klass, 'before_delayed_enqueue', *args)
+    end
+
+    def run_before_schedule_hooks(klass, *args)
+      run_hooks(klass, 'before_schedule', *args)
+    end
+
+    def run_after_schedule_hooks(klass, *args)
+      run_hooks(klass, 'after_schedule', *args)
     end
   end
 end

data/lib/resque_scheduler/server.rb

@@ -1,14 +1,12 @@
 require 'resque_scheduler'
 require 'resque/server'
+require 'json'
+
 # Extend Resque::Server to add tabs
 module ResqueScheduler
-
   module Server
-
     def self.included(base)
-
       base.class_eval do
-
         helpers do
           def format_time(t)
             t.strftime("%Y-%m-%d %H:%M:%S %z")
@@ -17,6 +15,41 @@ module ResqueScheduler
           def queue_from_class_name(class_name)
             Resque.queue_from_class(ResqueScheduler::Util.constantize(class_name))
           end
+
+          def schedule_interval(config)
+            if config['every']
+              schedule_interval_every(config['every'])
+            elsif config['cron']
+              'cron: ' + config['cron'].to_s
+            else
+              'Not currently scheduled'
+            end
+          end
+
+          def schedule_interval_every(every)
+            s = 'every: '
+            if every.respond_to?(:first)
+              s << every.first
+            else
+              s << every
+            end
+
+            return s unless every.respond_to?(:last) && every.length > 1
+
+            s << ' ('
+            meta = every.last.map do |key, value|
+              "#{key.to_s.gsub(/_/, ' ')} #{value}"
+            end
+            s << meta.join(', ') << ')'
+          end
+
+          def schedule_class(config)
+            if config['class'].nil? && !config['custom_job_class'].nil?
+              config['custom_job_class']
+            else
+              config['class']
+            end
+          end
         end
 
         get "/schedule" do
@@ -59,6 +92,18 @@ module ResqueScheduler
           erb File.read(File.join(File.dirname(__FILE__), 'server/views/delayed.erb'))
         end
 
+        get "/delayed/jobs/:klass" do
+          begin
+            klass = ResqueScheduler::Util::constantize(params[:klass])
+            @args = JSON.load(URI.decode(params[:args]))
+            @timestamps = Resque.scheduled_at(klass, *@args)
+          rescue => err
+            @timestamps = []
+          end
+
+          erb File.read(File.join(File.dirname(__FILE__), 'server/views/delayed_schedules.erb'))
+        end
+
         get "/delayed/:timestamp" do
           # Is there a better way to specify alternate template locations with sinatra?
           erb File.read(File.join(File.dirname(__FILE__), 'server/views/delayed_timestamp.erb'))
@@ -74,18 +119,14 @@ module ResqueScheduler
           Resque.reset_delayed_queue
           redirect u('delayed')
         end
-
       end
-
     end
-
-    Resque::Server.tabs << 'Schedule'
-    Resque::Server.tabs << 'Delayed'
-
   end
-
 end
 
+Resque::Server.tabs << 'Schedule'
+Resque::Server.tabs << 'Delayed'
+
 Resque::Server.class_eval do
   include ResqueScheduler::Server
 end