resque-scheduler 4.3.1 → 4.6.0

data/lib/resque/scheduler/delaying_extensions.rb

@@ -24,7 +24,7 @@ module Resque
       def enqueue_at_with_queue(queue, timestamp, klass, *args)
         return false unless plugin.run_before_schedule_hooks(klass, *args)
 
-        if Resque.inline? || timestamp.to_i < Time.now.to_i
+        if Resque.inline? || timestamp.to_i <= Time.now.to_i
           # Just create the job and let resque perform it right away with
           # inline.  If the class is a custom job class, call self#scheduled
           # on it. This allows you to do things like
@@ -33,7 +33,7 @@ module Resque
           if klass.respond_to?(:scheduled)
             klass.scheduled(queue, klass.to_s, *args)
           else
-            Resque::Job.create(queue, klass, *args)
+            Resque.enqueue_to(queue, klass, *args)
           end
         else
           delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
@@ -45,6 +45,9 @@ module Resque
       # Identical to enqueue_at but takes number_of_seconds_from_now
       # instead of a timestamp.
       def enqueue_in(number_of_seconds_from_now, klass, *args)
+        unless number_of_seconds_from_now.is_a?(Numeric)
+          raise ArgumentError, 'Please supply a numeric number of seconds'
+        end
         enqueue_at(Time.now + number_of_seconds_from_now, klass, *args)
       end
 
@@ -53,14 +56,35 @@ module Resque
       # number of seconds has passed.
       def enqueue_in_with_queue(queue, number_of_seconds_from_now,
                                 klass, *args)
+        unless number_of_seconds_from_now.is_a?(Numeric)
+          raise ArgumentError, 'Please supply a numeric number of seconds'
+        end
         enqueue_at_with_queue(queue, Time.now + number_of_seconds_from_now,
                               klass, *args)
       end
 
+      # Update the delayed timestamp of any matching delayed jobs or enqueue a
+      # new job if no matching jobs are found. Returns the number of delayed or
+      # enqueued jobs.
+      def delay_or_enqueue_at(timestamp, klass, *args)
+        count = remove_delayed(klass, *args)
+        count = 1 if count == 0
+
+        count.times do
+          enqueue_at(timestamp, klass, *args)
+        end
+      end
+
+      # Identical to +delay_or_enqueue_at+, except it takes
+      # number_of_seconds_from_now instead of a timestamp
+      def delay_or_enqueue_in(number_of_seconds_from_now, klass, *args)
+        delay_or_enqueue_at(Time.now + number_of_seconds_from_now, klass, *args)
+      end
+
       # Used internally to stuff the item into the schedule sorted list.
-      # +timestamp+ can be either in seconds or a datetime object Insertion
-      # if O(log(n)).  Returns true if it's the first job to be scheduled at
-      # that time, else false
+      # +timestamp+ can be either in seconds or a datetime object. The
+      # insertion time complexity is O(log(n)). Returns true if it's
+      # the first job to be scheduled at that time, else false.
       def delayed_push(timestamp, item)
         # First add this item to the list for this timestamp
         redis.rpush("delayed:#{timestamp.to_i}", encode(item))
@@ -82,6 +106,7 @@ module Resque
       end
 
       # Returns the size of the delayed queue schedule
+      # this does not represent the number of items in the queue to be scheduled
       def delayed_queue_schedule_size
         redis.zcard :delayed_queue_schedule
       end
@@ -128,10 +153,7 @@ module Resque
         Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |item|
           key = "delayed:#{item}"
           items = redis.lrange(key, 0, -1)
-          redis.pipelined do
-            items.each { |ts_item| redis.del("timestamps:#{ts_item}") }
-          end
-          redis.del key
+          redis.del(key, items.map { |ts_item| "timestamps:#{ts_item}" })
         end
 
         redis.del :delayed_queue_schedule
@@ -143,6 +165,11 @@ module Resque
         remove_delayed_job(search)
       end
 
+      def remove_delayed_in_queue(klass, queue, *args)
+        search = encode(job_to_hash_with_queue(queue, klass, args))
+        remove_delayed_job(search)
+      end
+
       # Given an encoded item, enqueue it now
       def enqueue_delayed(klass, *args)
         hash = job_to_hash(klass, args)
@@ -151,6 +178,13 @@ module Resque
         end
       end
 
+      def enqueue_delayed_with_queue(klass, queue, *args)
+        hash = job_to_hash_with_queue(queue, klass, args)
+        remove_delayed_in_queue(klass, queue, *args).times do
+          Resque::Scheduler.enqueue_from_config(hash)
+        end
+      end
+
       # Given a block, remove jobs that return true from a block
       #
       # This allows for removal of delayed jobs that have arguments matching
@@ -175,7 +209,15 @@ module Resque
         found_jobs.reduce(0) do |sum, encoded_job|
           decoded_job = decode(encoded_job)
           klass = Util.constantize(decoded_job['class'])
-          sum + enqueue_delayed(klass, *decoded_job['args'])
+          queue = decoded_job['queue']
+
+          if queue
+            jobs_queued = enqueue_delayed_with_queue(klass, queue, *decoded_job['args'])
+          else
+            jobs_queued = enqueue_delayed(klass, *decoded_job['args'])
+          end
+
+          jobs_queued + sum
         end
       end
 
@@ -190,9 +232,9 @@ module Resque
 
         # Beyond 100 there's almost no improvement in speed
         found = timestamps.each_slice(100).map do |ts_group|
-          jobs = redis.pipelined do |r|
+          jobs = redis.pipelined do |pipeline|
             ts_group.each do |ts|
-              r.lrange("delayed:#{ts}", 0, -1)
+              pipeline.lrange("delayed:#{ts}", 0, -1)
             end
           end
 
@@ -265,18 +307,23 @@ module Resque
         { class: klass.to_s, args: args, queue: queue }
       end
 
+      # Removes a job from the queue, but not modify the timestamp schedule. This method
+      # will not effect the output of `delayed_queue_schedule_size`
       def remove_delayed_job(encoded_job)
         return 0 if Resque.inline?
 
         timestamps = redis.smembers("timestamps:#{encoded_job}")
 
-        replies = redis.pipelined do
+        replies = redis.pipelined do |pipeline|
           timestamps.each do |key|
-            redis.lrem(key, 0, encoded_job)
-            redis.srem("timestamps:#{encoded_job}", key)
+            pipeline.lrem(key, 0, encoded_job)
+            pipeline.srem("timestamps:#{encoded_job}", key)
           end
         end
 
+        # timestamp key is not removed from the schedule, this is done later
+        # by the scheduler loop
+
         return 0 if replies.nil? || replies.empty?
         replies.each_slice(2).map(&:first).inject(:+)
       end
@@ -287,9 +334,9 @@ module Resque
         redis.watch(key) do
           if redis.llen(key).to_i == 0
             # If the list is empty, remove it.
-            redis.multi do
-              redis.del(key)
-              redis.zrem(:delayed_queue_schedule, timestamp.to_i)
+            redis.multi do |transaction|
+              transaction.del(key)
+              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
             end
           else
             redis.redis.unwatch

data/lib/resque/scheduler/env.rb

@@ -38,12 +38,8 @@ module Resque
                                     true
                                   end
 
-        unless Process.respond_to?('daemon')
-          abort 'background option is set, which requires ruby >= 1.9'
-        end
-
         Process.daemon(true, !Resque::Scheduler.quiet)
-        Resque.redis.client.reconnect
+        Resque.redis._client.reconnect
       end
 
       def setup_pid_file
@@ -64,13 +60,13 @@ module Resque
 
           c.dynamic = !!options[:dynamic] if options.key?(:dynamic)
 
-          c.env = options[:env] if options.key(:env)
+          c.env = options[:env] if options.key?(:env)
 
           c.logfile = options[:logfile] if options.key?(:logfile)
 
           c.logformat = options[:logformat] if options.key?(:logformat)
 
-          if psleep = options[:poll_sleep_amount] && !psleep.nil?
+          if (psleep = options[:poll_sleep_amount]) && !psleep.nil?
             c.poll_sleep_amount = Float(psleep)
           end
 

data/lib/resque/scheduler/lock/resilient.rb

@@ -43,7 +43,7 @@ module Resque
           @locked_sha = nil if refresh
 
           @locked_sha ||=
-            Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+            Resque.data_store.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
               if redis.call('GET', KEYS[1]) == ARGV[1]
               then
                 redis.call('EXPIRE', KEYS[1], #{timeout})
@@ -62,7 +62,7 @@ module Resque
           @acquire_sha = nil if refresh
 
           @acquire_sha ||=
-            Resque.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
+            Resque.data_store.redis.script(:load, <<-EOF.gsub(/^ {14}/, ''))
               if redis.call('SETNX', KEYS[1], ARGV[1]) == 1
               then
                 redis.call('EXPIRE', KEYS[1], #{timeout})

data/lib/resque/scheduler/locking.rb

@@ -2,7 +2,7 @@
 
 # ### Locking the scheduler process
 #
-# There are two places in resque-scheduler that need to be synchonized in order
+# There are two places in resque-scheduler that need to be synchronized in order
 # to be able to run redundant scheduler processes while ensuring jobs don't get
 # queued multiple times when the master process changes.
 #
@@ -76,7 +76,7 @@ module Resque
 
       def release_master_lock
         master_lock.release
-      rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError
+      rescue *INTERMITTENT_ERRORS
         @master_lock = nil
       end
 
@@ -97,7 +97,7 @@ module Resque
       end
 
       def redis_master_version
-        Resque.redis.info['redis_version'].to_f
+        Resque.data_store.redis.info['redis_version'].to_f
       end
     end
   end

data/lib/resque/scheduler/scheduling_extensions.rb

@@ -36,7 +36,7 @@ module Resque
       # :args can be any yaml which will be converted to a ruby literal and
       # passed in a params. (optional)
       #
-      # :rails_envs is the list of envs where the job gets loaded. Envs are
+      # :rails_env is the list of envs where the job gets loaded. Envs are
       # comma separated (optional)
       #
       # :description is just that, a description of the job (optional). If
@@ -101,12 +101,13 @@ module Resque
       end
 
       # remove a given schedule by name
-      def remove_schedule(name)
+      # Preventing a reload is optional and available to batch operations
+      def remove_schedule(name, reload = true)
         non_persistent_schedules.delete(name)
         redis.hdel(:persistent_schedules, name)
         redis.sadd(:schedules_changed, name)
 
-        reload_schedule!
+        reload_schedule! if reload
       end
 
       private

data/lib/resque/scheduler/server/views/delayed.erb

@@ -1,9 +1,9 @@
 <h1>Delayed Jobs</h1>
-<%- size = resque.delayed_queue_schedule_size %>
+<% size = resque.delayed_queue_schedule_size %>
 
 <%= scheduler_view :search_form, layout: false %>
 
-<p style="font-color: red; font-weight: bold;">
+<p style="color: red; font-weight: bold;">
   <%= @error_message %>
 </p>
 
@@ -46,7 +46,7 @@
       <td><%= h(show_job_arguments(job['args'])) if job && delayed_timestamp_size == 1 %></td>
       <td>
         <% if job %>
-          <a href="<%=u URI("/delayed/jobs/#{job['class']}?args=" + URI.encode(job['args'].to_json)) %>">All schedules</a>
+          <a href="<%= u URI("/delayed/jobs/#{CGI.escape(job['class'])}?args=" + CGI.escape(job['args'].to_json)) %>">All schedules</a>
         <% end %>
       </td>
     </tr>
@@ -55,7 +55,7 @@
 
 <% if size > 0 %>
   <br>
-  <form method="POST" action="<%=u 'delayed/clear'%>" class='clear-delayed'>
+  <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed'>
     <input type='submit' name='' value='Clear Delayed Jobs' />
   </form>
 <% end %>

data/lib/resque/scheduler/server/views/delayed_timestamp.erb

@@ -13,7 +13,7 @@
   <% jobs.each do |job| %>
     <tr>
       <td class='class'><%= job['class'] %></td>
-      <td class='args'><%=h show_job_arguments(job['args']) %></td>
+      <td class='args'><%= h show_job_arguments(job['args']) %></td>
     </tr>
   <% end %>
   <% if jobs.empty? %>

data/lib/resque/scheduler/server/views/scheduler.erb

@@ -8,7 +8,7 @@
   <br/> Current master: <%= Resque.redis.get(Resque::Scheduler.master_lock.key) %>
 </p>
 <p class='intro'>
-	The highlighted jobs are skipped for current environment.
+  The highlighted jobs are skipped for current environment.
 </p>
 <div style="overflow-y: auto; width:100%; padding: 0px 5px;">
 <table>
@@ -29,7 +29,7 @@
   <% Resque.schedule.keys.sort.each_with_index do |name, index| %>
     <% config = Resque.schedule[name] %>
     <tr style="<%= scheduled_in_this_env?(name) ? '' : 'color: #9F6000;background: #FEEFB3;' %>">
-	    <td style="padding-left: 15px;"><%= index+ 1 %>.</td>
+      <td style="padding-left: 15px;"><%= index + 1 %>.</td>
       <% if Resque::Scheduler.dynamic %>
         <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/schedule" %>" method="post" style="margin-left: 0">

data/lib/resque/scheduler/server/views/search.erb

@@ -33,7 +33,6 @@
       </tr>
   <% end %>
 </table>
-</h1>
 
 <% queued = @jobs.select { |j| j['where_at'] == 'queued' } %>
 <h1>Queued jobs</h1>
@@ -68,5 +67,3 @@
   </tr>
   <% end %>
 </table>
-
-

data/lib/resque/scheduler/server/views/search_form.erb

@@ -1,8 +1,4 @@
 <form method="POST" action="<%= u 'delayed/search' %>">
-  <input type='input' name='search' value="<%= params[:search] %>"/>
+  <input type='input' name='search' value="<%= h params[:search] %>"/>
   <input type='submit' value='Search'/>
 </form>
-
-
-
-

data/lib/resque/scheduler/server.rb

@@ -87,7 +87,7 @@ module Resque
         def delayed_jobs_klass
           begin
             klass = Resque::Scheduler::Util.constantize(params[:klass])
-            @args = JSON.load(URI.decode(params[:args]))
+            @args = JSON.load(CGI.unescape(params[:args]))
             @timestamps = Resque.scheduled_at(klass, *@args)
           rescue
             @timestamps = []

data/lib/resque/scheduler/signal_handling.rb

@@ -10,13 +10,13 @@ module Resque
       end
 
       # For all signals, set the shutdown flag and wait for current
-      # poll/enqueing to finish (should be almost instant).  In the
+      # poll/enqueuing to finish (should be almost instant).  In the
       # case of sleeping, exit immediately.
       def register_signal_handlers
         (Signal.list.keys & %w(INT TERM USR1 USR2 QUIT)).each do |sig|
           trap(sig) do
             signal_queue << sig
-            # break sleep in the primary scheduler thread, alowing
+            # break sleep in the primary scheduler thread, allowing
             # the signal queue to get processed as soon as possible.
             @th.wakeup if @th && @th.alive?
           end

data/lib/resque/scheduler/util.rb

@@ -4,7 +4,7 @@ module Resque
   module Scheduler
     class Util
       # In order to upgrade to resque(1.25) which has deprecated following
-      # methods, we just added these usefull helpers back to use in Resque
+      # methods, we just added these useful helpers back to use in Resque
       # Scheduler.  refer to:
       # https://github.com/resque/resque-scheduler/pull/273
 

data/lib/resque/scheduler/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Scheduler
-    VERSION = '4.3.1'.freeze
+    VERSION = '4.6.0'.freeze
   end
 end

data/lib/resque/scheduler.rb

@@ -1,5 +1,6 @@
 # vim:fileencoding=utf-8
 
+require 'redis/errors'
 require 'rufus/scheduler'
 require_relative 'scheduler/configuration'
 require_relative 'scheduler/locking'
@@ -13,6 +14,9 @@ module Resque
     autoload :Extension, 'resque/scheduler/extension'
     autoload :Util, 'resque/scheduler/util'
     autoload :VERSION, 'resque/scheduler/version'
+    INTERMITTENT_ERRORS = [
+      Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError, Redis::TimeoutError
+    ].freeze
 
     private
 
@@ -44,13 +48,7 @@ module Resque
         $stdout.sync = true
         $stderr.sync = true
 
-        # Load the schedule into rufus
-        # If dynamic is set, load that schedule otherwise use normal load
-        if dynamic
-          reload_schedule!
-        else
-          load_schedule!
-        end
+        was_master = nil
 
         begin
           @th = Thread.current
@@ -58,11 +56,21 @@ module Resque
           # Now start the scheduling part of the loop.
           loop do
             begin
-              if master?
+              # Check on changes to master/child
+              @am_master = master?
+              if am_master != was_master
+                procline am_master ? 'Master scheduler' : 'Child scheduler'
+
+                # Load schedule because changed
+                reload_schedule!
+              end
+
+              if am_master
                 handle_delayed_items
                 update_schedule if dynamic
               end
-            rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError => e
+              was_master = am_master
+            rescue *INTERMITTENT_ERRORS => e
               log! e.message
               release_master_lock
             end
@@ -99,7 +107,7 @@ module Resque
         Resque.schedule.each do |name, config|
           load_schedule_job(name, config)
         end
-        Resque.redis.del(:schedules_changed)
+        Resque.redis.del(:schedules_changed) if am_master && dynamic
         procline 'Schedules Loaded'
       end
 
@@ -202,7 +210,7 @@ module Resque
         loop do
           handle_shutdown do
             # Continually check that it is still the master
-            item = enqueue_next_item(timestamp) if master?
+            item = enqueue_next_item(timestamp) if am_master
           end
           # continue processing until there are no more ready items in this
           # timestamp
@@ -243,7 +251,7 @@ module Resque
         if job_klass && job_klass != 'Resque::Job'
           # The custom job class API must offer a static "scheduled" method. If
           # the custom job class can not be constantized (via a requeue call
-          # from the web perhaps), fall back to enqueing normally via
+          # from the web perhaps), fall back to enqueuing normally via
           # Resque::Job.create.
           begin
             Resque::Scheduler::Util.constantize(job_klass).scheduled(
@@ -369,7 +377,6 @@ module Resque
 
       def stop_rufus_scheduler
         rufus_scheduler.shutdown(:wait)
-        rufus_scheduler.join
       end
 
       def before_shutdown
@@ -420,10 +427,10 @@ module Resque
       private
 
       def enqueue_recurring(name, config)
-        if master?
+        if am_master
           log! "queueing #{config['class']} (#{name})"
-          Resque.last_enqueued_at(name, Time.now.to_s)
           enqueue(config)
+          Resque.last_enqueued_at(name, Time.now.to_s)
         end
       end
 
@@ -442,6 +449,11 @@ module Resque
       def internal_name
         "resque-scheduler-#{Resque::Scheduler::VERSION}"
       end
+
+      def am_master
+        @am_master = master? unless defined?(@am_master)
+        @am_master
+      end
     end
   end
 end

data/resque-scheduler.gemspec

@@ -11,12 +11,14 @@ Gem::Specification.new do |spec|
     Simon Eskildsen
     Ryan Biesemeyer
     Dan Buch
+    Michael Bianco
   EOF
   spec.email = %w(
     bvandenbos@gmail.com
     sirup@sirupsen.com
     ryan@yaauie.com
     dan@meatballhat.com
+    mike@mikebian.co
   )
   spec.summary = 'Light weight job scheduling on top of Resque'
   spec.description = <<-DESCRIPTION
@@ -24,13 +26,16 @@ Gem::Specification.new do |spec|
     Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
     Also supports queueing jobs on a fixed, cron-like schedule.
   DESCRIPTION
-  spec.homepage = 'http://github.com/resque/resque-scheduler'
+  spec.homepage = 'https://github.com/resque/resque-scheduler'
   spec.license = 'MIT'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.required_ruby_version = '>= 2.3.0'
 
   spec.files = `git ls-files -z`.split("\0").reject do |f|
     f.match(%r{^(test|spec|features|examples|bin|tasks)/}) ||
-      f.match(/^(Vagrantfile|Gemfile\.lock|appveyor\.yml)/) ||
-      f.match(/^\.(rubocop|simplecov|travis|vagrant|gitignore)/)
+      f.match(/^(Vagrantfile|Gemfile\.lock)/) ||
+      f.match(/^\.(rubocop|simplecov|vagrant|gitignore)/)
   end
   spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -38,7 +43,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'json'
-  spec.add_development_dependency 'kramdown'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
@@ -47,14 +51,16 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'yard'
-  spec.add_development_dependency 'tzinfo-data'
+  spec.add_development_dependency 'timecop'
 
   # We pin rubocop because new cops have a tendency to result in false-y
   # positives for new contributors, which is not a nice experience.
   spec.add_development_dependency 'rubocop', '~> 0.40.0'
 
   spec.add_runtime_dependency 'mono_logger', '~> 1.0'
-  spec.add_runtime_dependency 'redis', '>= 3.3', '< 5'
-  spec.add_runtime_dependency 'resque', '~> 1.26'
-  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2'
+  spec.add_runtime_dependency 'redis', '>= 3.3'
+  spec.add_runtime_dependency 'resque', '>= 1.27'
+  # rufus-scheduler v3.7 causes a failure in test/multi_process_test.rb
+  # rufus-scheduler v3.3 is missing a to_local method which fails tests
+  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2', '!= 3.3'
 end