resque-scheduler 2.0.1 → 2.1.0

data/Rakefile

@@ -1,28 +1,41 @@
-require 'bundler'
-
-Bundler::GemHelper.install_tasks
+require 'bundler/gem_tasks'
 
 $LOAD_PATH.unshift 'lib'
 
 task :default => :test
 
-# Tests
-desc "Run tests"
+desc 'Run tests'
 task :test do
+  if RUBY_VERSION =~ /^1\.8/
+    unless ENV['SEED']
+      srand
+      ENV['SEED'] = (srand % 0xFFFF).to_s
+    end
+
+    $stdout.puts "Running with SEED=#{ENV['SEED']}"
+    srand Integer(ENV['SEED'])
+  elsif ENV['SEED']
+    ARGV += %W(--seed #{ENV['SEED']})
+  end
   Dir['test/*_test.rb'].each do |f|
     require File.expand_path(f)
   end
 end
 
-# Documentation Tasks
+desc 'Run rubocop'
+task :rubocop do
+  unless RUBY_VERSION < '1.9'
+    sh('rubocop --config .rubocop.yml --format simple') { |r, _| r || abort }
+  end
+end
+
 begin
   require 'rdoc/task'
 
   Rake::RDocTask.new do |rd|
-    rd.main = "README.markdown"
-    rd.rdoc_files.include("README.markdown", "lib/**/*.rb")
+    rd.main = 'README.md'
+    rd.rdoc_files.include('README.md', 'lib/**/*.rb')
     rd.rdoc_dir = 'doc'
   end
 rescue LoadError
 end
-

data/lib/resque/scheduler.rb

@@ -1,12 +1,11 @@
 require 'rufus/scheduler'
-require 'thwait'
 require 'resque/scheduler_locking'
+require 'resque_scheduler/logger_builder'
 
 module Resque
 
   class Scheduler
 
-    extend Resque::Helpers
     extend Resque::SchedulerLocking
 
     class << self
@@ -17,6 +16,9 @@ module Resque
       # If set, produces no output
       attr_accessor :mute
 
+      # If set, will write messages to the file
+      attr_accessor :logfile
+
       # If set, will try to update the schedule in the loop
       attr_accessor :dynamic
 
@@ -24,6 +26,8 @@ module Resque
       # queue.  Defaults to 5
       attr_writer :poll_sleep_amount
 
+      attr_writer :logger
+
       # the Rufus::Scheduler jobs that are scheduled
       def scheduled_jobs
         @@scheduled_jobs
@@ -33,12 +37,22 @@ module Resque
         @poll_sleep_amount ||= 5 # seconds
       end
 
+      def logger
+        @logger ||= ResqueScheduler::LoggerBuilder.new(:mute => mute, :verbose => verbose, :log_dev => logfile).build
+      end
+
       # Schedule all jobs and continually look for delayed jobs (never returns)
       def run
         $0 = "resque-scheduler: Starting"
         # trap signals
         register_signal_handlers
 
+        # Quote from the resque/worker.
+        # Fix buffering so we can `rake resque:scheduler > scheduler.log` and
+        # get output from the child in there.
+        $stdout.sync = true
+        $stderr.sync = true
+
         # Load the schedule into rufus
         # If dynamic is set, load that schedule otherwise use normal load
         if dynamic
@@ -62,7 +76,7 @@ module Resque
 
         # never gets here.
       end
-     
+
 
       # For all signals, set the shutdown flag and wait for current
       # poll/enqueing to finish (should be almost istant).  In the
@@ -205,7 +219,7 @@ module Resque
         args = job_config['args'] || job_config[:args]
 
         klass_name = job_config['class'] || job_config[:class]
-        klass = constantize(klass_name) rescue klass_name
+        klass = Resque.constantize(klass_name) rescue klass_name
 
         params = args.is_a?(Hash) ? [args] : Array(args)
         queue = job_config['queue'] || job_config[:queue] || Resque.queue_from_class(klass)
@@ -215,7 +229,7 @@ module Resque
           # job class can not be constantized (via a requeue call from the web perhaps), fall
           # back to enqueing normally via Resque::Job.create.
           begin
-            constantize(job_klass).scheduled(queue, klass_name, *params)
+            Resque.constantize(job_klass).scheduled(queue, klass_name, *params)
           rescue NameError
             # Note that the custom job class (job_config['custom_job_class']) is the one enqueued
             Resque::Job.create(queue, job_klass, *params)
@@ -297,18 +311,17 @@ module Resque
       def shutdown
         @shutdown = true
         if @sleeping
-          release_master_lock!
+          Thread.new { release_master_lock! }
           exit
         end
       end
 
       def log!(msg)
-        puts "#{Time.now.strftime("%Y-%m-%d %H:%M:%S")} #{msg}" unless mute
+        logger.info msg
       end
 
       def log(msg)
-        # add "verbose" logic later
-        log!(msg) if verbose
+        logger.debug msg
       end
 
       def procline(string)

data/lib/resque_scheduler.rb

@@ -25,6 +25,9 @@ module ResqueScheduler
   # is used implicitly as "class" argument - in the "MakeTea" example,
   # "MakeTea" is used both as job name and resque worker class.
   #
+  # Any jobs that were in the old schedule, but are not 
+  # present in the new schedule, will be removed.
+  #
   # :cron can be any cron scheduling string
   #
   # :every can be used in lieu of :cron. see rufus-scheduler's 'every' usage
@@ -46,9 +49,13 @@ module ResqueScheduler
     schedule_hash = prepare_schedule(schedule_hash)
 
     if Resque::Scheduler.dynamic
+      reload_schedule!
       schedule_hash.each do |name, job_spec|
         set_schedule(name, job_spec)
       end
+      (schedule.keys - schedule_hash.keys.map(&:to_s)).each do |name|
+        remove_schedule(name)
+      end
     end
     @schedule = schedule_hash
   end
@@ -121,7 +128,7 @@ module ResqueScheduler
   def enqueue_at_with_queue(queue, timestamp, klass, *args)
     return false unless Plugin.run_before_schedule_hooks(klass, *args)
 
-    if Resque.inline?
+    if Resque.inline? || timestamp.to_i < Time.now.to_i
       # Just create the job and let resque perform it right away with inline.
       # If the class is a custom job class, call self#scheduled on it. This allows you to do things like
       # Resque.enqueue_at(timestamp, CustomJobClass, :opt1 => val1). Otherwise, pass off to Resque.
@@ -158,6 +165,9 @@ module ResqueScheduler
     # First add this item to the list for this timestamp
     redis.rpush("delayed:#{timestamp.to_i}", encode(item))
 
+    # Store the timestamps at with this item occurs
+    redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
+
     # Now, add this timestamp to the zsets.  The score and the value are
     # the same since we'll be querying by timestamp, and we don't have
     # anything else to store.
@@ -166,7 +176,7 @@ module ResqueScheduler
 
   # Returns an array of timestamps based on start and count
   def delayed_queue_peek(start, count)
-    Array(redis.zrange(:delayed_queue_schedule, start, start+count-1)).collect{|x| x.to_i}
+    Array(redis.zrange(:delayed_queue_schedule, start, start+count-1)).collect { |x| x.to_i }
   end
 
   # Returns the size of the delayed queue schedule
@@ -203,7 +213,9 @@ module ResqueScheduler
   def next_item_for_timestamp(timestamp)
     key = "delayed:#{timestamp.to_i}"
 
-    item = decode redis.lpop(key)
+    encoded_item = redis.lpop(key)
+    redis.srem("timestamps:#{encoded_item}", key)
+    item = decode(encoded_item)
 
     # If the list is empty, remove it.
     clean_up_timestamp(key, timestamp)
@@ -213,24 +225,36 @@ module ResqueScheduler
   # Clears all jobs created with enqueue_at or enqueue_in
   def reset_delayed_queue
     Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |item|
-      redis.del "delayed:#{item}"
+      key = "delayed:#{item}"
+      items = redis.lrange(key, 0, -1)
+      redis.pipelined do
+        items.each { |ts_item| redis.del("timestamps:#{ts_item}") }
+      end
+      redis.del key
     end
 
     redis.del :delayed_queue_schedule
   end
 
   # Given an encoded item, remove it from the delayed_queue
-  #
-  # This method is potentially very expensive since it needs to scan
-  # through the delayed queue for every timestamp, but at least it
-  # doesn't kill Redis by calling redis.keys.
   def remove_delayed(klass, *args)
-    destroyed = 0
     search = encode(job_to_hash(klass, args))
-    Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |timestamp|
-      destroyed += redis.lrem "delayed:#{timestamp}", 0, search
+    timestamps = redis.smembers("timestamps:#{search}")
+
+    replies = redis.pipelined do
+      timestamps.each do |key|
+        redis.lrem(key, 0, search)
+        redis.srem("timestamps:#{search}", key)
+      end
     end
-    destroyed
+
+    (replies.nil? || replies.empty?) ? 0 : replies.each_slice(2).collect { |slice| slice.first }.inject(:+)
+  end
+
+  # Given an encoded item, enqueue it now
+  def enqueue_delayed(klass, *args)
+    hash = job_to_hash(klass, args)
+    remove_delayed(klass, *args).times { Resque::Scheduler.enqueue_from_config(hash) }
   end
 
   # Given a timestamp and job (klass + args) it removes all instances and
@@ -240,8 +264,12 @@ module ResqueScheduler
   # timestamp
   def remove_delayed_job_from_timestamp(timestamp, klass, *args)
     key = "delayed:#{timestamp.to_i}"
-    count = redis.lrem key, 0, encode(job_to_hash(klass, args))
+    encoded_job = encode(job_to_hash(klass, args))
+
+    redis.srem("timestamps:#{encoded_job}", key)
+    count = redis.lrem(key, 0, encoded_job)
     clean_up_timestamp(key, timestamp)
+
     count
   end
 
@@ -253,6 +281,14 @@ module ResqueScheduler
     total_jobs
   end
 
+  # Returns delayed jobs schedule timestamp for +klass+, +args+.
+  def scheduled_at(klass, *args)
+    search = encode(job_to_hash(klass, args))
+    redis.smembers("timestamps:#{search}").collect do |key|
+      key.tr('delayed:', '').to_i
+    end
+  end
+
   private
 
     def job_to_hash(klass, args)

data/lib/resque_scheduler/logger_builder.rb

@@ -0,0 +1,51 @@
+module ResqueScheduler
+  # Just builds a logger, with specified verbosity and destination.
+  # The simplest example:
+  #
+  #   ResqueScheduler::LoggerBuilder.new.build
+  class LoggerBuilder
+    # Initializes new instance of the builder
+    #
+    # Pass :opts Hash with
+    #   - :mute if logger needs to be silent for all levels. Default - false
+    #   - :verbose if there is a need in debug messages. Default - false
+    #   - :log_dev to output logs into a desired file. Default - STDOUT
+    #
+    # Example:
+    #
+    #   LoggerBuilder.new(:mute => false, :verbose => true, :log_dev => 'log/sheduler.log')
+    def initialize(opts={})
+      @muted   = !!opts[:mute]
+      @verbose = !!opts[:verbose]
+      @log_dev = opts[:log_dev] || STDOUT
+    end
+
+    # Returns an instance of Logger
+    def build
+      logger = Logger.new(@log_dev)
+      logger.level = level
+      logger.datetime_format = "%Y-%m-%d %H:%M:%S"
+      logger.formatter = formatter
+
+      logger
+    end
+
+    private
+
+    def level
+      if @verbose && !@muted
+        Logger::DEBUG
+      elsif !@muted
+        Logger::INFO
+      else
+        Logger::FATAL
+      end
+    end
+
+    def formatter
+      proc do |severity, datetime, progname, msg|
+        "[#{severity}] #{datetime}: #{msg}\n"
+      end
+    end
+  end
+end

data/lib/resque_scheduler/server/views/delayed.erb

@@ -8,6 +8,7 @@
 
 <p class='intro'>
   This list below contains the timestamps for scheduled delayed jobs.
+  Server local time: <%= Time.now %>
 </p>
 
 <p class='sub'>
@@ -45,4 +46,4 @@
   <% end %>
 </table>
 
-<%= partial :next_more, :start => start, :size => size %>
+<%= partial :next_more, :start => start, :size => size %>

data/lib/resque_scheduler/server/views/scheduler.erb

@@ -3,6 +3,7 @@
 <p class='intro'>
   The list below contains all scheduled jobs.  Click &quot;Queue now&quot; to queue
   a job immediately.
+  Server local time: <%= Time.now %>
 </p>
 
 <table>
@@ -27,9 +28,9 @@
       <td><%= h name %></td>
       <td><%= h config['description'] %></td>
       <td style="white-space:nowrap"><%= if !config['every'].nil? 
-                                           h('every: ' + config['every'])
+                                           h('every: ' + (config['every'].is_a?(Array) ? config['every'].join(", ") : config['every'].to_s ))
                                          elsif !config['cron'].nil?
-                                           h('cron: ' + config['cron']) 
+                                           h('cron: ' + config['cron'].to_s) 
                                          else
                                            'Not currently scheduled'
                                          end %></td>

data/lib/resque_scheduler/tasks.rb

@@ -9,17 +9,23 @@ namespace :resque do
     require 'resque'
     require 'resque_scheduler'
 
+    # Need to set this here for conditional Process.daemon redirect of stderr/stdout to /dev/null
+    Resque::Scheduler.mute = true if ENV['MUTE']
+
     if ENV['BACKGROUND']
       unless Process.respond_to?('daemon')
         abort "env var BACKGROUND is set, which requires ruby >= 1.9"
       end
-      Process.daemon(true)
+      Process.daemon(true, !Resque::Scheduler.mute)
+      Resque.redis.client.reconnect
     end
 
     File.open(ENV['PIDFILE'], 'w') { |f| f << Process.pid.to_s } if ENV['PIDFILE']
 
-    Resque::Scheduler.dynamic = true if ENV['DYNAMIC_SCHEDULE']
-    Resque::Scheduler.verbose = true if ENV['VERBOSE']
+    Resque::Scheduler.dynamic           = true if ENV['DYNAMIC_SCHEDULE']
+    Resque::Scheduler.verbose           = true if ENV['VERBOSE']
+    Resque::Scheduler.logfile           = ENV['LOGFILE'] if ENV['LOGFILE']
+    Resque::Scheduler.poll_sleep_amount = Integer(ENV['INTERVAL']) if ENV['INTERVAL']
     Resque::Scheduler.run
   end
 

data/lib/resque_scheduler/version.rb

@@ -1,3 +1,3 @@
 module ResqueScheduler
-  VERSION = '2.0.1'
+  VERSION = '2.1.0'
 end

data/resque-scheduler.gemspec

@@ -1,27 +1,32 @@
 # -*- encoding: utf-8 -*-
-$:.unshift File.expand_path("../lib", __FILE__)
-require "resque_scheduler/version"
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'resque_scheduler/version'
 
-Gem::Specification.new do |s|
-  s.name        = "resque-scheduler"
-  s.version     = ResqueScheduler::VERSION
-  s.platform    = Gem::Platform::RUBY
-  s.authors     = ['Ben VandenBos']
-  s.email       = ['bvandenbos@gmail.com']
-  s.homepage    = "http://github.com/bvandenbos/resque-scheduler"
-  s.summary     = "Light weight job scheduling on top of Resque"
-  s.description = %q{Light weight job scheduling on top of Resque.
+Gem::Specification.new do |spec|
+  spec.name        = 'resque-scheduler'
+  spec.version     = ResqueScheduler::VERSION
+  spec.authors     = ['Ben VandenBos']
+  spec.email       = ['bvandenbos@gmail.com']
+  spec.homepage    = 'http://github.com/resque/resque-scheduler'
+  spec.summary     = 'Light weight job scheduling on top of Resque'
+  spec.description = %q{Light weight job scheduling on top of Resque.
     Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
     Also supports queueing jobs on a fixed, cron-like schedule.}
 
-  s.required_rubygems_version = ">= 1.3.6"
-  s.add_development_dependency "bundler", ">= 1.0.0"
+  spec.files        = `git ls-files`.split("\n")
+  spec.executables  = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files   = spec.files.grep(%r{^test/})
+  spec.require_path = ['lib']
 
-  s.files        = `git ls-files`.split("\n")
-  s.executables  = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact
-  s.require_path = 'lib'
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'mocha'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'json' if RUBY_VERSION < '1.9'
+  spec.add_development_dependency 'rubocop' unless RUBY_VERSION < '1.9'
 
-  s.add_runtime_dependency(%q<redis>, [">= 2.0.1"])
-  s.add_runtime_dependency(%q<resque>, [">= 1.20.0"])
-  s.add_runtime_dependency(%q<rufus-scheduler>, [">= 0"])
+  spec.add_runtime_dependency 'redis', '>= 2.0.1'
+  spec.add_runtime_dependency 'resque', ['>= 1.20.0', '< 1.25']
+  spec.add_runtime_dependency 'rufus-scheduler', '>= 0'
 end