resque-scheduler 1.9.10 → 2.0.0.a

data/.gitignore

@@ -1,6 +1,3 @@
 .bundle/
-doc/
 pkg
 nbproject
-Gemfile.lock
-.rvmrc

data/Gemfile

@@ -1,3 +1,2 @@
-source 'https://rubygems.org'
-
+source :rubygems
 gemspec

data/Gemfile.lock

@@ -0,0 +1,52 @@
+PATH
+  remote: .
+  specs:
+    resque-scheduler (1.9.7)
+      redis (>= 2.0.1)
+      resque (>= 1.8.0)
+      rufus-scheduler
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    git (1.2.5)
+    jeweler (1.5.1)
+      bundler (~> 1.0.0)
+      git (>= 1.2.5)
+      rake
+    json (1.4.6)
+    mocha (0.9.9)
+      rake
+    rack (1.2.1)
+    rack-test (0.5.6)
+      rack (>= 1.0)
+    rake (0.8.7)
+    redis (2.1.1)
+    redis-namespace (0.8.0)
+      redis (< 3.0.0)
+    resque (1.10.0)
+      json (~> 1.4.6)
+      redis-namespace (~> 0.8.0)
+      sinatra (>= 0.9.2)
+      vegas (~> 0.1.2)
+    rufus-scheduler (2.0.7)
+      tzinfo
+    sinatra (1.1.0)
+      rack (~> 1.1)
+      tilt (~> 1.1)
+    tilt (1.1)
+    tzinfo (0.3.23)
+    vegas (0.1.8)
+      rack (>= 1.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  jeweler
+  mocha
+  rack-test
+  redis (>= 2.0.1)
+  resque (>= 1.8.0)
+  resque-scheduler!
+  rufus-scheduler

data/HISTORY.md

@@ -1,18 +1,7 @@
-## 1.9.10 (2013-09-19)
+## 2.0.0 (???)
 
-* Backported `#enqueue_at_with_queue`
-* Locking to resque < 1.25.0
-* Mocha setup compatibility
-* Ruby 1.8 compatibility in scheduler tab when schedule keys are symbols
-
-## 1.9.9 (2011-03-29)
-
-* Compatibility with resque 1.15.0
-
-## 1.9.8 (???)
-
-* Validates delayed jobs prior to insertion into the delayed queue (bogdan)
-* Rescue exceptions that occur during queuing and log them (dgrijalva)
+* Dynamic schedule support (brianjlandau, davidyang)
+* Now depends on redis >=1.3
 
 ## 1.9.7 (2010-11-09)
 

data/README.markdown

@@ -4,7 +4,7 @@ resque-scheduler
 Resque-scheduler is an extension to [Resque](http://github.com/defunkt/resque)
 that adds support for queueing items in the future.
 
-Requires redis >=1.1.
+Requires redis >=1.3.
 
 
 Job scheduling is supported in two different way:
@@ -30,9 +30,9 @@ is most likely stored in a YAML like so:
       args: contributors
       description: "This job resets the weekly leaderboard for contributions"
 
-A queue option can also be specified. When job will go onto the specified queue
-if it is available (Even if @queue is specified in the job class). When the
-queue is given it is not necessary for the scheduler to load the class.
+A queue option can also be specified. Then the job will go onto the specified
+queue if it is available (Even if @queue is specified in the job class). When
+the queue is given it is not necessary for the scheduler to load the class.
 
     clear_leaderboards_moderator:
       cron: "30 6 * * 1"
@@ -95,6 +95,86 @@ If you have the need to cancel a delayed job, you can do so thusly:
     # remove the job with exactly the same parameters:
     Resque.remove_delayed(SendFollowUpEmail, :user_id => current_user.id)
 
+
+### Dynamic Schedules
+
+If needed you can also have recurring jobs (scheduled) that are dynamically
+defined and updated inside of your application.  A good example is if you want
+to allow users to configured when a report is automatically generated.  This
+can be completed by loading the schedule initially wherever you configure
+Resque and setting `Resque::Scheduler.dynamic` to `true`. Then subsequently
+updating the "`schedules`" key in redis, namespaced to the Resque namespace.
+The "`schedules`" key is expected to be a redis hash data type, where the key
+is the name of the schedule and the value is a JSON encoded hash of the
+schedule configuration.  There are methods on Resque to make this easy (see
+below).
+
+When the scheduler loops it will look for differences between the existing
+schedule and the current schedule in redis. If there are differences it will
+make the necessary changes to the running schedule. The schedule names that
+need to be changed are stored in the `schedules_changed` set in redis.
+
+To force the scheduler to reload the schedule you just send it the `USR2`
+signal.  This will force a complete schedule reload (unscheduling and
+rescheduling everything).
+
+To add/update, delete, and retrieve individual schedule items you should
+use the provided API methods:
+
+* `Resque.set_schedule(name, config)`
+* `Resque.get_schedule(name)`
+* `Resque.remove_schedule(name)`
+
+For example:
+
+    Resque.set_schedule("create_fake_leaderboards", {
+      :cron => "30 6 * * 1",
+      :class => "CreateFakeLeaderboards",
+      :queue => scoring
+    })
+
+In this way, it's possible to completely configure your scheduled jobs from
+inside your app if you so desire.
+
+### Support for customized Job classes
+
+Some Resque extensions like
+[resque-status](http://github.com/quirkey/resque-status) use custom job
+classes with a slightly different API signature.  Resque-scheduler isn't
+trying to support all existing and future custom job classes, instead it
+supports a schedule flag so you can extend your custom class and make it
+support scheduled job.
+
+Let's pretend we have a JobWithStatus class called FakeLeaderboard
+
+		class FakeLeaderboard < Resque::JobWithStatus
+			def perform
+				# do something and keep track of the status
+			end
+		end
+
+    create_fake_leaderboards:
+      cron: "30 6 * * 1"
+      queue: scoring
+      custom_job_class: FakeLeaderboard
+      args: 
+      rails_env: demo
+      description: "This job will auto-create leaderboards for our online demo and the status will update as the worker makes progress"
+
+If your extension doesn't support scheduled job, you would need to extend the
+custom job class to support the #scheduled method:
+
+    module Resque
+      class JobWithStatus
+        # Wrapper API to forward a Resque::Job creation API call into
+        # a JobWithStatus call.
+        def self.scheduled(queue, klass, *args)
+          create(args)
+        end
+      end
+    end
+
+
 ### Schedule jobs per environment
 
 Resque-Scheduler allows to create schedule jobs for specific envs.  The arg
@@ -138,39 +218,6 @@ Multiple envs are allowed, separated by commas:
 NOTE: If you specify the `rails_env` arg without setting RAILS_ENV as an 
 environment variable, the job won't be loaded.
 
-### Support for customized Job classes
-
-Some Resque extensions like [resque-status](http://github.com/quirkey/resque-status) use custom job classes with a slightly different API signature.
-Resque-scheduler isn't trying to support all existing and future custom job classes, instead it supports a schedule flag so you can extend your custom class
-and make it support scheduled job.
-
-Let's pretend we have a JobWithStatus class called FakeLeaderboard
-
-		class FakeLeaderboard < Resque::JobWithStatus
-			def perform
-				# do something and keep track of the status
-			end
-		end
-
-    create_fake_leaderboards:
-      cron: "30 6 * * 1"
-      queue: scoring
-      custom_job_class: FakeLeaderboard
-      args: 
-      rails_env: demo
-      description: "This job will auto-create leaderboards for our online demo and the status will update as the worker makes progress"
-
-If your extension doesn't support scheduled job, you would need to extend the custom job class to support the #scheduled method:
-
-    module Resque
-      class JobWithStatus
-        # Wrapper API to forward a Resque::Job creation API call into a JobWithStatus call.
-        def self.scheduled(queue, klass, *args)
-          create(args)
-        end
-      end
-    end
-
 
 Resque-web additions
 --------------------
@@ -188,8 +235,8 @@ The Delayed tab:
 ![The Delayed Tab](http://img.skitch.com/20100111-ne4fcqtc5emkcuwc5qtais2kwx.jpg)
 
 Get get these to show up you need to pass a file to `resque-web` to tell it to
-include the `resque-scheduler` plugin.  You probably already have a file somewhere
-where you configure `resque`.  It probably looks something like this:
+include the `resque-scheduler` plugin.  You probably already have a file
+somewhere where you configure `resque`.  It probably looks something like this:
 
     require 'resque' # include resque so we can configure it
     Resque.redis = "redis_server:6379" # tell Resque where redis lives
@@ -198,7 +245,10 @@ Now, you want to add the following:
 
     require 'resque_scheduler' # include the resque_scheduler (this makes the tabs show up)
 
-And if you have a schedule you want to set, add this:
+As of resque-scheduler 2.0, it's no longer necessary to have the resque-web
+process aware of the schedule because it reads it from redis.  But prior to
+2.0, you'll want to make sure you load the schedule in this file as well.
+Something like this:
 
     Resque.schedule = YAML.load_file(File.join(RAILS_ROOT, 'config/resque_schedule.yml')) # load the schedule
 
@@ -209,7 +259,6 @@ Now make sure you're passing that file to resque-web like so:
 That should make the scheduler tabs show up in `resque-web`.
 
 
-
 Installation and the Scheduler process
 --------------------------------------
 
@@ -217,10 +266,15 @@ To install:
 
     gem install resque-scheduler
 
-You'll need to add this to your rakefile:
+The unless you specify the `queue` for each scheduled job, the scheduler 
+needs to know about your job classes (so it can put them into the appropriate
+queue).  To do so, extend the "resque:scheduler_setup" to load your app's code.
+In rails, it would look something like this:
 
     require 'resque_scheduler/tasks'
-    task "resque:setup" => :environment
+    task "resque:scheduler_setup" => :environment # load the env so we know about the job classes
+
+By default, "resque:scheduler_setup" invokes "resque:setup".
 
 The scheduler process is just a rake task which is responsible for both queueing
 items from the schedule and polling the delayed queue for items ready to be
@@ -230,7 +284,7 @@ pushed on to the work queues.  For obvious reasons, this process never exits.
 
 Supported environment variables are `VERBOSE` and `MUTE`.  If either is set to
 any nonempty value, they will take effect.  `VERBOSE` simply dumps more output
-to stdout.  `MUTE` does the opposite and silences all output. `MUTE` supercedes
+to stdout.  `MUTE` does the opposite and silences all output. `MUTE` supersedes
 `VERBOSE`.
 
 NOTE: You DO NOT want to run >1 instance of the scheduler.  Doing so will result

data/Rakefile

@@ -1,41 +1,13 @@
-require 'bundler/gem_tasks'
+require 'bundler'
+Bundler::GemHelper.install_tasks
 
 $LOAD_PATH.unshift 'lib'
 
 task :default => :test
 
-desc 'Run tests'
+desc "Run tests"
 task :test do
-  if RUBY_VERSION =~ /^1\.8/
-    unless ENV['SEED']
-      srand
-      ENV['SEED'] = (srand % 0xFFFF).to_s
-    end
-
-    $stdout.puts "Running with SEED=#{ENV['SEED']}"
-    srand Integer(ENV['SEED'])
-  elsif ENV['SEED']
-    ARGV += %W(--seed #{ENV['SEED']})
-  end
   Dir['test/*_test.rb'].each do |f|
     require File.expand_path(f)
   end
 end
-
-desc 'Run rubocop'
-task :rubocop do
-  unless RUBY_VERSION < '1.9'
-    sh('rubocop --config .rubocop.yml --format simple') { |r, _| r || abort }
-  end
-end
-
-begin
-  require 'rdoc/task'
-
-  Rake::RDocTask.new do |rd|
-    rd.main = 'README.md'
-    rd.rdoc_files.include('README.md', 'lib/**/*.rb')
-    rd.rdoc_dir = 'doc'
-  end
-rescue LoadError
-end

data/lib/resque/scheduler.rb

@@ -11,22 +11,32 @@ module Resque
 
       # If true, logs more stuff...
       attr_accessor :verbose
-
+      
       # If set, produces no output
       attr_accessor :mute
+      
+      # If set, will try to update the schulde in the loop
+      attr_accessor :dynamic
+      
+      # the Rufus::Scheduler jobs that are scheduled
+      def scheduled_jobs
+        @@scheduled_jobs
+      end
 
       # Schedule all jobs and continually look for delayed jobs (never returns)
       def run
-
+        $0 = "resque-scheduler: Starting"
         # trap signals
         register_signal_handlers
 
         # Load the schedule into rufus
+        procline "Loading Schedule"
         load_schedule!
 
         # Now start the scheduling part of the loop.
         loop do
           handle_delayed_items
+          update_schedule if dynamic
           poll_sleep
         end
 
@@ -39,12 +49,13 @@ module Resque
       def register_signal_handlers
         trap("TERM") { shutdown }
         trap("INT") { shutdown }
-
+        
         begin
           trap('QUIT') { shutdown   }
           trap('USR1') { kill_child }
+          trap('USR2') { reload_schedule! }
         rescue ArgumentError
-          warn "Signals QUIT and USR1 not supported."
+          warn "Signals QUIT and USR1 and USR2 not supported."
         end
       end
 
@@ -52,29 +63,42 @@ module Resque
       # rufus scheduler instance
       def load_schedule!
         log! "Schedule empty! Set Resque.schedule" if Resque.schedule.empty?
-
+        
+        @@scheduled_jobs = {}
+        
         Resque.schedule.each do |name, config|
-          # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
-          # required for the jobs to be scheduled.  If rails_env is missing, the
-          # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
-          # to.
-          if config['rails_env'].nil? || rails_env_matches?(config)
-            log! "Scheduling #{name} "
-            interval_defined = false
-            interval_types = %w{cron every}
-            interval_types.each do |interval_type|
-              if !config[interval_type].nil? && config[interval_type].length > 0
-                rufus_scheduler.send(interval_type, config[interval_type]) do
+          load_schedule_job(name, config)
+        end
+        Resque.redis.del(:schedules_changed)
+        procline "Schedules Loaded"
+      end
+      
+      # Loads a job schedule into the Rufus::Scheduler and stores it in @@scheduled_jobs
+      def load_schedule_job(name, config)
+        # If rails_env is set in the config, enforce ENV['RAILS_ENV'] as
+        # required for the jobs to be scheduled.  If rails_env is missing, the
+        # job should be scheduled regardless of what ENV['RAILS_ENV'] is set
+        # to.
+        if config['rails_env'].nil? || rails_env_matches?(config)
+          log! "Scheduling #{name} "
+          interval_defined = false
+          interval_types = %w{cron every}
+          interval_types.each do |interval_type|
+            if !config[interval_type].nil? && config[interval_type].length > 0
+              begin
+                @@scheduled_jobs[name] = rufus_scheduler.send(interval_type, config[interval_type]) do
                   log! "queueing #{config['class']} (#{name})"
                   enqueue_from_config(config)
                 end
-                interval_defined = true
-                break
+              rescue Exception => e
+                log! "#{e.class.name}: #{e.message}"
               end
+              interval_defined = true
+              break
             end
-            unless interval_defined
-              log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
-            end
+          end
+          unless interval_defined
+            log! "no #{interval_types.join(' / ')} found for #{config['class']} (#{name}) - skipping"
           end
         end
       end
@@ -88,33 +112,30 @@ module Resque
       # Handles queueing delayed items
       # at_time - Time to start scheduling items (default: now).
       def handle_delayed_items(at_time=nil)
-        timestamp = nil
-        begin
-          if timestamp = Resque.next_delayed_timestamp(at_time)
+        item = nil
+        if timestamp = Resque.next_delayed_timestamp(at_time)
+          procline "Processing Delayed Items"
+          while !timestamp.nil?
             enqueue_delayed_items_for_timestamp(timestamp)
+            timestamp = Resque.next_delayed_timestamp(at_time)
           end
-        # continue processing until there are no more ready timestamps
-        end while !timestamp.nil?
+        end
       end
-
+      
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
         item = nil
         begin
           handle_shutdown do
             if item = Resque.next_item_for_timestamp(timestamp)
-              begin
-                log "queuing #{item['class']} [delayed]"
-                queue = item['queue'] || Resque.queue_from_class(constantize(item['class']))
-                # Support custom job classes like job with status
-                if (job_klass = item['custom_job_class']) && (job_klass != 'Resque::Job')
-                  # custom job classes not supporting the same API calls must implement the #schedule method
-                  constantize(job_klass).scheduled(queue, item['class'], *item['args'])
-                else
-                  Resque::Job.create(queue, item['class'], *item['args'])
-                end
-              rescue
-                log! "Failed to enqueue #{klass_name}:\n #{$!}"
+              log "queuing #{item['class']} [delayed]"
+              queue = item['queue'] || Resque.queue_from_class(constantize(item['class']))
+              # Support custom job classes like job with status
+              if (job_klass = item['custom_job_class']) && (job_klass != 'Resque::Job')
+                # custom job classes not supporting the same API calls must implement the #schedule method
+                constantize(job_klass).scheduled(queue, item['class'], *item['args'])
+              else
+                Resque::Job.create(queue, item['class'], *item['args'])
               end
             end
           end
@@ -140,9 +161,7 @@ module Resque
           constantize(job_klass).scheduled(queue, klass_name, *params)
         else
           Resque::Job.create(queue, klass_name, *params)
-        end
-      rescue
-        log! "Failed to enqueue #{klass_name}:\n #{$!}"
+        end        
       end
 
       def rufus_scheduler
@@ -154,8 +173,40 @@ module Resque
       def clear_schedule!
         rufus_scheduler.stop
         @rufus_scheduler = nil
+        @@scheduled_jobs = {}
         rufus_scheduler
       end
+      
+      def reload_schedule!
+        procline "Reloading Schedule"
+        clear_schedule!
+        Resque.reload_schedule!
+        load_schedule!
+      end
+      
+      def update_schedule
+        if Resque.redis.scard(:schedules_changed) > 0
+          procline "Updating schedule"
+          Resque.reload_schedule!
+          while schedule_name = Resque.redis.spop(:schedules_changed)
+            if Resque.schedule.keys.include?(schedule_name)
+              unschedule_job(schedule_name)
+              load_schedule_job(schedule_name, Resque.schedule[schedule_name])
+            else
+              unschedule_job(schedule_name)
+            end
+          end
+          procline "Schedules Loaded"
+        end
+      end
+      
+      def unschedule_job(name)
+        if scheduled_jobs[name]
+          log "Removing schedule #{name}"
+          scheduled_jobs[name].unschedule
+          @@scheduled_jobs.delete(name)
+        end
+      end
 
       # Sleeps and returns true
       def poll_sleep
@@ -179,6 +230,11 @@ module Resque
         # add "verbose" logic later
         log!(msg) if verbose
       end
+      
+      def procline(string)
+        $0 = "resque-scheduler-#{ResqueScheduler::Version}: #{string}"
+        log! $0
+      end
 
     end