resque-scheduler 4.4.0 → 4.11.0

data/lib/resque/scheduler/configuration.rb

@@ -8,13 +8,18 @@ module Resque
         yield self
       end
 
+      attr_writer :environment
+      def environment
+        @environment ||= ENV
+      end
+
       # Used in `#load_schedule_job`
       attr_writer :env
 
       def env
         return @env if @env
         @env ||= Rails.env if defined?(Rails) && Rails.respond_to?(:env)
-        @env ||= ENV['RAILS_ENV']
+        @env ||= environment['RAILS_ENV']
         @env
       end
 
@@ -22,42 +27,48 @@ module Resque
       attr_writer :verbose
 
       def verbose
-        @verbose ||= !!ENV['VERBOSE']
+        @verbose ||= to_bool(environment['VERBOSE'])
       end
 
       # If set, produces no output
       attr_writer :quiet
 
       def quiet
-        @quiet ||= !!ENV['QUIET']
+        @quiet ||= to_bool(environment['QUIET'])
       end
 
       # If set, will write messages to the file
       attr_writer :logfile
 
       def logfile
-        @logfile ||= ENV['LOGFILE']
+        @logfile ||= environment['LOGFILE']
       end
 
-      # Sets whether to log in 'text' or 'json'
+      # Sets whether to log in 'text', 'json' or 'logfmt'
       attr_writer :logformat
 
       def logformat
-        @logformat ||= ENV['LOGFORMAT']
+        @logformat ||= environment['LOGFORMAT']
       end
 
       # If set, will try to update the schedule in the loop
       attr_writer :dynamic
 
       def dynamic
-        @dynamic ||= !!ENV['DYNAMIC_SCHEDULE']
+        @dynamic ||= to_bool(environment['DYNAMIC_SCHEDULE'])
       end
 
       # If set, will append the app name to procline
       attr_writer :app_name
 
       def app_name
-        @app_name ||= ENV['APP_NAME']
+        @app_name ||= environment['APP_NAME']
+      end
+
+      def delayed_requeue_batch_size
+        @delayed_requeue_batch_size ||= \
+          ENV['DELAYED_REQUEUE_BATCH_SIZE'].to_i if environment['DELAYED_REQUEUE_BATCH_SIZE']
+        @delayed_requeue_batch_size ||= 100
       end
 
       # Amount of time in seconds to sleep between polls of the delayed
@@ -66,7 +77,32 @@ module Resque
 
       def poll_sleep_amount
         @poll_sleep_amount ||=
-          Float(ENV.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
+          Float(environment.fetch('RESQUE_SCHEDULER_INTERVAL', '5'))
+      end
+
+      # Sets timeout for Resque::Scheduler::Lock::Base
+      attr_writer :lock_timeout
+
+      def lock_timeout
+        @lock_timeout ||= environment.fetch('LOCK_TIMEOUT', 60 * 3).to_i
+      end
+
+      private
+
+      # Copied from https://github.com/rails/rails/blob/main/activemodel/lib/active_model/type/boolean.rb#L17
+      TRUE_VALUES = [
+        true, 1,
+        '1', :'1',
+        't', :t,
+        'T', :T,
+        'true', :true,
+        'TRUE', :TRUE,
+        'on', :on,
+        'ON', :ON
+      ].to_set.freeze
+
+      def to_bool(value)
+        TRUE_VALUES.include?(value)
       end
     end
   end

data/lib/resque/scheduler/delaying_extensions.rb

@@ -22,24 +22,22 @@ module Resque
       # timestamp has passed. It respects Resque.inline option, by
       # creating the job right away instead of adding to the queue.
       def enqueue_at_with_queue(queue, timestamp, klass, *args)
-        return false unless plugin.run_before_schedule_hooks(klass, *args)
-
-        if Resque.inline? || timestamp.to_i < Time.now.to_i
-          # Just create the job and let resque perform it right away with
-          # inline.  If the class is a custom job class, call self#scheduled
-          # on it. This allows you to do things like
-          # Resque.enqueue_at(timestamp, CustomJobClass, :opt1 => val1).
-          # Otherwise, pass off to Resque.
-          if klass.respond_to?(:scheduled)
-            klass.scheduled(queue, klass.to_s, *args)
+        plugin.process_schedule_hooks(klass, *args) do
+          if Resque.inline? || timestamp.to_i <= Time.now.to_i
+            # Just create the job and let resque perform it right away with
+            # inline.  If the class is a custom job class, call self#scheduled
+            # on it. This allows you to do things like
+            # Resque.enqueue_at(timestamp, CustomJobClass, :opt1 => val1).
+            # Otherwise, pass off to Resque.
+            if klass.respond_to?(:scheduled)
+              klass.scheduled(queue, klass.to_s, *args)
+            else
+              Resque.enqueue_to(queue, klass, *args)
+            end
           else
-            Resque::Job.create(queue, klass, *args)
+            delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
           end
-        else
-          delayed_push(timestamp, job_to_hash_with_queue(queue, klass, args))
         end
-
-        plugin.run_after_schedule_hooks(klass, *args)
       end
 
       # Identical to enqueue_at but takes number_of_seconds_from_now
@@ -63,16 +61,34 @@ module Resque
                               klass, *args)
       end
 
+      # Update the delayed timestamp of any matching delayed jobs or enqueue a
+      # new job if no matching jobs are found. Returns the number of delayed or
+      # enqueued jobs.
+      def delay_or_enqueue_at(timestamp, klass, *args)
+        count = remove_delayed(klass, *args)
+        count = 1 if count == 0
+
+        count.times do
+          enqueue_at(timestamp, klass, *args)
+        end
+      end
+
+      # Identical to +delay_or_enqueue_at+, except it takes
+      # number_of_seconds_from_now instead of a timestamp
+      def delay_or_enqueue_in(number_of_seconds_from_now, klass, *args)
+        delay_or_enqueue_at(Time.now + number_of_seconds_from_now, klass, *args)
+      end
+
       # Used internally to stuff the item into the schedule sorted list.
-      # +timestamp+ can be either in seconds or a datetime object Insertion
-      # if O(log(n)).  Returns true if it's the first job to be scheduled at
-      # that time, else false
+      # +timestamp+ can be either in seconds or a datetime object. The
+      # insertion time complexity is O(log(n)). Returns true if it's
+      # the first job to be scheduled at that time, else false.
       def delayed_push(timestamp, item)
         # First add this item to the list for this timestamp
         redis.rpush("delayed:#{timestamp.to_i}", encode(item))
 
         # Store the timestamps at with this item occurs
-        redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
+        redis.sadd("timestamps:#{encode(item)}", ["delayed:#{timestamp.to_i}"])
 
         # Now, add this timestamp to the zsets.  The score and the value are
         # the same since we'll be querying by timestamp, and we don't have
@@ -88,6 +104,7 @@ module Resque
       end
 
       # Returns the size of the delayed queue schedule
+      # this does not represent the number of items in the queue to be scheduled
       def delayed_queue_schedule_size
         redis.zcard :delayed_queue_schedule
       end
@@ -121,7 +138,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
 
         encoded_item = redis.lpop(key)
-        redis.srem("timestamps:#{encoded_item}", key)
+        redis.srem("timestamps:#{encoded_item}", [key])
         item = decode(encoded_item)
 
         # If the list is empty, remove it.
@@ -134,10 +151,7 @@ module Resque
         Array(redis.zrange(:delayed_queue_schedule, 0, -1)).each do |item|
           key = "delayed:#{item}"
           items = redis.lrange(key, 0, -1)
-          redis.pipelined do
-            items.each { |ts_item| redis.del("timestamps:#{ts_item}") }
-          end
-          redis.del key
+          redis.del(key, items.map { |ts_item| "timestamps:#{ts_item}" })
         end
 
         redis.del :delayed_queue_schedule
@@ -149,6 +163,11 @@ module Resque
         remove_delayed_job(search)
       end
 
+      def remove_delayed_in_queue(klass, queue, *args)
+        search = encode(job_to_hash_with_queue(queue, klass, args))
+        remove_delayed_job(search)
+      end
+
       # Given an encoded item, enqueue it now
       def enqueue_delayed(klass, *args)
         hash = job_to_hash(klass, args)
@@ -157,6 +176,13 @@ module Resque
         end
       end
 
+      def enqueue_delayed_with_queue(klass, queue, *args)
+        hash = job_to_hash_with_queue(queue, klass, args)
+        remove_delayed_in_queue(klass, queue, *args).times do
+          Resque::Scheduler.enqueue_from_config(hash)
+        end
+      end
+
       # Given a block, remove jobs that return true from a block
       #
       # This allows for removal of delayed jobs that have arguments matching
@@ -181,7 +207,15 @@ module Resque
         found_jobs.reduce(0) do |sum, encoded_job|
           decoded_job = decode(encoded_job)
           klass = Util.constantize(decoded_job['class'])
-          sum + enqueue_delayed(klass, *decoded_job['args'])
+          queue = decoded_job['queue']
+
+          if queue
+            jobs_queued = enqueue_delayed_with_queue(klass, queue, *decoded_job['args'])
+          else
+            jobs_queued = enqueue_delayed(klass, *decoded_job['args'])
+          end
+
+          jobs_queued + sum
         end
       end
 
@@ -196,9 +230,9 @@ module Resque
 
         # Beyond 100 there's almost no improvement in speed
         found = timestamps.each_slice(100).map do |ts_group|
-          jobs = redis.pipelined do |r|
+          jobs = redis.pipelined do |pipeline|
             ts_group.each do |ts|
-              r.lrange("delayed:#{ts}", 0, -1)
+              pipeline.lrange("delayed:#{ts}", 0, -1)
             end
           end
 
@@ -221,7 +255,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
         encoded_job = encode(job_to_hash(klass, args))
 
-        redis.srem("timestamps:#{encoded_job}", key)
+        redis.srem("timestamps:#{encoded_job}", [key])
         count = redis.lrem(key, 0, encoded_job)
         clean_up_timestamp(key, timestamp)
 
@@ -261,6 +295,22 @@ module Resque
         redis.hget('delayed:last_enqueued_at', job_name)
       end
 
+      def clean_up_timestamp(key, timestamp)
+        # Use a watch here to ensure nobody adds jobs to this delayed
+        # queue while we're removing it.
+        redis.watch(key) do
+          if redis.llen(key).to_i == 0
+            # If the list is empty, remove it.
+            redis.multi do |transaction|
+              transaction.del(key)
+              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
+            end
+          else
+            redis.redis.unwatch
+          end
+        end
+      end
+
       private
 
       def job_to_hash(klass, args)
@@ -271,38 +321,27 @@ module Resque
         { class: klass.to_s, args: args, queue: queue }
       end
 
+      # Removes a job from the queue, but not modify the timestamp schedule. This method
+      # will not effect the output of `delayed_queue_schedule_size`
       def remove_delayed_job(encoded_job)
         return 0 if Resque.inline?
 
         timestamps = redis.smembers("timestamps:#{encoded_job}")
 
-        replies = redis.pipelined do
+        replies = redis.pipelined do |pipeline|
           timestamps.each do |key|
-            redis.lrem(key, 0, encoded_job)
-            redis.srem("timestamps:#{encoded_job}", key)
+            pipeline.lrem(key, 0, encoded_job)
+            pipeline.srem("timestamps:#{encoded_job}", [key])
           end
         end
 
+        # timestamp key is not removed from the schedule, this is done later
+        # by the scheduler loop
+
         return 0 if replies.nil? || replies.empty?
         replies.each_slice(2).map(&:first).inject(:+)
       end
 
-      def clean_up_timestamp(key, timestamp)
-        # Use a watch here to ensure nobody adds jobs to this delayed
-        # queue while we're removing it.
-        redis.watch(key) do
-          if redis.llen(key).to_i == 0
-            # If the list is empty, remove it.
-            redis.multi do
-              redis.del(key)
-              redis.zrem(:delayed_queue_schedule, timestamp.to_i)
-            end
-          else
-            redis.redis.unwatch
-          end
-        end
-      end
-
       def search_first_delayed_timestamp_in_range(start_at, stop_at)
         start_at = start_at.nil? ? '-inf' : start_at.to_i
         stop_at = stop_at.nil? ? '+inf' : stop_at.to_i

data/lib/resque/scheduler/env.rb

@@ -38,12 +38,8 @@ module Resque
                                     true
                                   end
 
-        unless Process.respond_to?('daemon')
-          abort 'background option is set, which requires ruby >= 1.9'
-        end
-
         Process.daemon(true, !Resque::Scheduler.quiet)
-        Resque.redis._client.reconnect
+        Resque.redis.reconnect
       end
 
       def setup_pid_file
@@ -58,6 +54,7 @@ module Resque
         at_exit { cleanup_pid_file }
       end
 
+      # rubocop:disable Metrics/AbcSize
       def setup_scheduler_configuration
         Resque::Scheduler.configure do |c|
           c.app_name = options[:app_name] if options.key?(:app_name)
@@ -70,6 +67,8 @@ module Resque
 
           c.logformat = options[:logformat] if options.key?(:logformat)
 
+          c.lock_timeout = options[:lock_timeout] if options.key?(:lock_timeout)
+
           if (psleep = options[:poll_sleep_amount]) && !psleep.nil?
             c.poll_sleep_amount = Float(psleep)
           end
@@ -77,6 +76,7 @@ module Resque
           c.verbose = !!options[:verbose] if options.key?(:verbose)
         end
       end
+      # rubocop:enable Metrics/AbcSize
 
       def cleanup_pid_file
         return unless pidfile_path

data/lib/resque/scheduler/lock/base.rb

@@ -11,7 +11,7 @@ module Resque
           @key = key
 
           # 3 minute default timeout
-          @timeout = options[:timeout] || 60 * 3
+          @timeout = options[:timeout] || Resque::Scheduler.lock_timeout
         end
 
         # Attempts to acquire the lock. Returns true if successfully acquired.
@@ -47,7 +47,7 @@ module Resque
 
         def hostname
           local_hostname = Socket.gethostname
-          Socket.gethostbyname(local_hostname).first
+          Addrinfo.getaddrinfo(local_hostname, 'http').first.getnameinfo.first
         rescue
           local_hostname
         end

data/lib/resque/scheduler/locking.rb

@@ -2,7 +2,7 @@
 
 # ### Locking the scheduler process
 #
-# There are two places in resque-scheduler that need to be synchonized in order
+# There are two places in resque-scheduler that need to be synchronized in order
 # to be able to run redundant scheduler processes while ensuring jobs don't get
 # queued multiple times when the master process changes.
 #

data/lib/resque/scheduler/logger_builder.rb

@@ -15,7 +15,7 @@ module Resque
       #   - :quiet if logger needs to be silent for all levels. Default - false
       #   - :verbose if there is a need in debug messages. Default - false
       #   - :log_dev to output logs into a desired file. Default - STDOUT
-      #   - :format log format, either 'text' or 'json'. Default - 'text'
+      #   - :format log format, either 'text', 'json' or 'logfmt'. Default - 'text'
       #
       # Example:
       #
@@ -32,6 +32,7 @@ module Resque
       # Returns an instance of MonoLogger
       def build
         logger = MonoLogger.new(@log_dev)
+        logger.progname = 'resque-scheduler'.freeze
         logger.level = level
         logger.formatter = send(:"#{@format}_formatter")
         logger
@@ -50,21 +51,31 @@ module Resque
       end
 
       def text_formatter
-        proc do |severity, datetime, _progname, msg|
-          "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
+        proc do |severity, datetime, progname, msg|
+          "#{progname}: [#{severity}] #{datetime.iso8601}: #{msg}\n"
         end
       end
 
       def json_formatter
         proc do |severity, datetime, progname, msg|
           require 'json'
-          JSON.dump(
-            name: 'resque-scheduler',
+          log_data = {
+            name: progname,
             progname: progname,
             level: severity,
             timestamp: datetime.iso8601,
             msg: msg
-          ) + "\n"
+          }
+          JSON.dump(log_data) + "\n"
+        end
+      end
+
+      def logfmt_formatter
+        proc do |severity, datetime, progname, msg|
+          "timestamp=\"#{datetime.iso8601}\" " \
+          "level=\"#{severity}\" " \
+          "progname=\"#{progname}\" " \
+          "msg=\"#{msg}\"\n"
         end
       end
     end

data/lib/resque/scheduler/plugin.rb

@@ -3,6 +3,15 @@
 module Resque
   module Scheduler
     module Plugin
+      def self.process_schedule_hooks(klass, *args)
+        # the documentation states that if any before_schedule hook returns
+        # false, the process should not be enqueued
+        return unless run_before_schedule_hooks(klass, *args)
+
+        yield
+        run_after_schedule_hooks(klass, *args)
+      end
+
       def self.hooks(job, pattern)
         job.methods.grep(/^#{pattern}/).sort
       end

data/lib/resque/scheduler/scheduling_extensions.rb

@@ -36,7 +36,7 @@ module Resque
       # :args can be any yaml which will be converted to a ruby literal and
       # passed in a params. (optional)
       #
-      # :rails_envs is the list of envs where the job gets loaded. Envs are
+      # :rails_env is the list of envs where the job gets loaded. Envs are
       # comma separated (optional)
       #
       # :description is just that, a description of the job (optional). If
@@ -91,7 +91,7 @@ module Resque
           non_persistent_schedules[name] = decode(encode(config))
         end
 
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
         reload_schedule! if reload
       end
 
@@ -101,12 +101,13 @@ module Resque
       end
 
       # remove a given schedule by name
-      def remove_schedule(name)
+      # Preventing a reload is optional and available to batch operations
+      def remove_schedule(name, reload = true)
         non_persistent_schedules.delete(name)
         redis.hdel(:persistent_schedules, name)
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
 
-        reload_schedule!
+        reload_schedule! if reload
       end
 
       private

data/lib/resque/scheduler/server/views/delayed.erb

@@ -1,9 +1,9 @@
 <h1>Delayed Jobs</h1>
-<%- size = resque.delayed_queue_schedule_size %>
+<% size = resque.delayed_queue_schedule_size %>
 
 <%= scheduler_view :search_form, layout: false %>
 
-<p style="font-color: red; font-weight: bold;">
+<p style="color: red; font-weight: bold;">
   <%= @error_message %>
 </p>
 
@@ -16,6 +16,14 @@
   Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%= size %></b> timestamps
 </p>
 
+<% if size > 0 %>
+  <div style="padding-bottom: 10px">
+    <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed confirmSubmission'>
+      <input type='submit' name='' value='Clear Delayed Jobs' />
+    </form>
+  </div>
+<% end %>
+
 <table>
   <tr>
     <th></th>
@@ -27,7 +35,7 @@
   </tr>
   <% resque.delayed_queue_peek(start, 20).each do |timestamp| %>
     <tr>
-      <td>
+      <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
         <form action="<%= u "/delayed/queue_now" %>" method="post">
           <input type="hidden" name="timestamp" value="<%= timestamp.to_i %>">
           <input type="submit" value="Queue now">
@@ -46,18 +54,11 @@
       <td><%= h(show_job_arguments(job['args'])) if job && delayed_timestamp_size == 1 %></td>
       <td>
         <% if job %>
-          <a href="<%=u URI("/delayed/jobs/#{URI.escape(job['class'])}?args=" + URI.encode(job['args'].to_json)) %>">All schedules</a>
+          <a href="<%= u URI("/delayed/jobs/#{CGI.escape(job['class'])}?args=" + CGI.escape(job['args'].to_json)) %>">All schedules</a>
         <% end %>
       </td>
     </tr>
   <% end %>
 </table>
 
-<% if size > 0 %>
-  <br>
-  <form method="POST" action="<%=u 'delayed/clear'%>" class='clear-delayed'>
-    <input type='submit' name='' value='Clear Delayed Jobs' />
-  </form>
-<% end %>
-
 <%= partial :next_more, :start => start, :size => size %>

data/lib/resque/scheduler/server/views/delayed_schedules.erb

@@ -1,4 +1,4 @@
-<h1>Delayed jobs scheduled for <%= params[:klass] %> (<%= show_job_arguments(@args) %>)</h1>
+<h1>Delayed jobs scheduled for <%=h params[:klass] %> (<%=h show_job_arguments(@args) %>)</h1>
 
 <table class='jobs'>
 <tr>

data/lib/resque/scheduler/server/views/delayed_timestamp.erb

@@ -13,7 +13,7 @@
   <% jobs.each do |job| %>
     <tr>
       <td class='class'><%= job['class'] %></td>
-      <td class='args'><%=h show_job_arguments(job['args']) %></td>
+      <td class='args'><%= h show_job_arguments(job['args']) %></td>
     </tr>
   <% end %>
   <% if jobs.empty? %>

data/lib/resque/scheduler/server/views/scheduler.erb

@@ -8,7 +8,7 @@
   <br/> Current master: <%= Resque.redis.get(Resque::Scheduler.master_lock.key) %>
 </p>
 <p class='intro'>
-	The highlighted jobs are skipped for current environment.
+  The highlighted jobs are skipped for current environment.
 </p>
 <div style="overflow-y: auto; width:100%; padding: 0px 5px;">
 <table>
@@ -29,7 +29,7 @@
   <% Resque.schedule.keys.sort.each_with_index do |name, index| %>
     <% config = Resque.schedule[name] %>
     <tr style="<%= scheduled_in_this_env?(name) ? '' : 'color: #9F6000;background: #FEEFB3;' %>">
-	    <td style="padding-left: 15px;"><%= index+ 1 %>.</td>
+      <td style="padding-left: 15px;"><%= index + 1 %>.</td>
       <% if Resque::Scheduler.dynamic %>
         <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/schedule" %>" method="post" style="margin-left: 0">

data/lib/resque/scheduler/server/views/search.erb

@@ -13,13 +13,13 @@
   </tr>
   <% delayed.each do |job| %>
       <tr>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/queue_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="submit" value="Queue now">
           </form>
         </td>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/cancel_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="hidden" name="klass" value="<%= job['class'] %>">
@@ -33,7 +33,6 @@
       </tr>
   <% end %>
 </table>
-</h1>
 
 <% queued = @jobs.select { |j| j['where_at'] == 'queued' } %>
 <h1>Queued jobs</h1>
@@ -68,5 +67,3 @@
   </tr>
   <% end %>
 </table>
-
-

data/lib/resque/scheduler/server/views/search_form.erb

@@ -1,8 +1,4 @@
 <form method="POST" action="<%= u 'delayed/search' %>">
-  <input type='input' name='search' value="<%= params[:search] %>"/>
+  <input type='input' name='search' value="<%= h params[:search] %>"/>
   <input type='submit' value='Search'/>
 </form>
-
-
-
-

data/lib/resque/scheduler/server.rb

@@ -87,7 +87,7 @@ module Resque
         def delayed_jobs_klass
           begin
             klass = Resque::Scheduler::Util.constantize(params[:klass])
-            @args = JSON.load(URI.decode(params[:args]))
+            @args = JSON.load(CGI.unescape(params[:args]))
             @timestamps = Resque.scheduled_at(klass, *@args)
           rescue
             @timestamps = []