resque-scheduler 4.8.0 → 4.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd9d3813b4dfbf31849dd4cc5866f8039face2673ec14c69906b90bb40f2208e
-  data.tar.gz: f3f4c9bfb5cb1a68c11a7284b8820a4231640a1ccf18c2a661777f51b2dc4f6b
+  metadata.gz: e3fea20c1e19d0477785e35cd8e568ed5edb31a1b3e5cb8b50b3bf6ceb8ed417
+  data.tar.gz: a015f8b83e75f674c2c74b477a656233a19ecc69ae10bbd8b1eca0a7d6328f23
 SHA512:
-  metadata.gz: 8ce3e99af96433cbc202163b2bea73ec9ee54a4f3691e987582bb110cfece6be0f53935e755d95d6113024e73cacaa23ae397dc4178e8e5b452553752add531f
-  data.tar.gz: b782b127a76f17c22c3299f90d57812eb966e9a7e7bcd1372d44d0204fbbab86c3f70e0fea1abb9181f8fb825f99bded51e6ff9f0c7358c1e59b6cfc04b1a604
+  metadata.gz: 2847caf20385af6e97a6316d4e1dc9c69841948a43d913cfe430c7a45786924315e9cf1b8b57c5bd37aa26e68f34fcfd35c6289fd9d63e233bbc0071ce962251
+  data.tar.gz: 8d612a72c7a3b671125e03d2dd6738691e5a9abce0d2b9ef8060f312b5793570ccc6052541740a07e0b67760359c8a72c4fe5690cca8ae9216322eb65fbabe9c

data/.github/workflows/rubocop.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         ruby: [
-            2.4
+            2.7
         ]
 
     steps:

data/.github/workflows/ruby.yml

@@ -26,6 +26,7 @@ jobs:
           - 2.7
           - "3.0"
           - 3.1
+          - 3.2
         resque-version:
           - "master"
           - "~> 2.4.0"
@@ -41,6 +42,8 @@ jobs:
         exclude:
           - ruby-version: head
             rufus-scheduler: 3.2
+          - ruby-version: 3.2
+            rufus-scheduler: 3.2
 
           - ruby-version: 2.3
             resque-version: "~> 1.27"

data/AUTHORS.md

@@ -29,6 +29,7 @@ Resque Scheduler authors
 - Henrik Nyh
 - Hormoz Kheradmand
 - Ian Davies
+- Irving Reid
 - James Le Cuirot
 - Jarkko Mönkkönen
 - Jimmy Chao
@@ -87,4 +88,5 @@ Resque Scheduler authors
 - malomalo
 - sawanoboly
 - serek
-- iloveitaly
+- iloveitaly
+- treacher

data/CHANGELOG.md

@@ -2,6 +2,23 @@
 
 **ATTN**: This project uses [semantic versioning](http://semver.org/).
 
+## [4.10.0] - 2023-08-20
+### Added
+* Add logfmt option for logging output (#763)
+
+### Fixed
+* Rubocop Fixes (#771)
+
+## [4.9.0] - 2023-05-31
+### Changed
+- Adding batching to re-queuing for timestamp by @brennen-stripe in #767
+- Use non-deprecated form of Redis.sadd/srem by @irvingreid in #752
+- Prompt for confirmation on 'Clear Delayed Jobs' by @mishina2228 in #754
+- Address some deprecation warnings in the test suite and with srem/lrem pipelined usage by @PatrickTulskie in #770
+
+### Fixed
+- Update CI matrix and fix tests by @mishina2228 in #766
+
 ## [4.8.0] - 2023-27-1
 - Replace deprecated Socket.gethostname with Addrinfo.getaddrinfo to fix deprecation warnings (#753)
 

data/Gemfile

@@ -24,4 +24,6 @@ else
   gem 'redis', redis_version
 end
 
+gem 'sinatra', '> 2.0'
+
 gemspec

data/README.md

@@ -139,7 +139,7 @@ requiring `resque` and `resque/scheduler` (default empty).
 * `RESQUE_SCHEDULER_INTERVAL` - Interval in seconds for checking if a
 scheduled job must run (coerced with `Kernel#Float()`) (default `5`)
 * `LOGFILE` - Log file name (default empty, meaning `$stdout`)
-* `LOGFORMAT` - Log output format to use (either `'text'` or `'json'`,
+* `LOGFORMAT` - Log output format to use (either `'text'`, `'json'` or `'logfmt'`,
 default `'text'`)
 * `PIDFILE` - If non-empty, write process PID to file (default empty)
 * `QUIET` - Silence most output if non-empty (equivalent to a level of
@@ -663,7 +663,7 @@ are toggled by environment variables:
   - `QUIET` will stop logging anything. Completely silent.
   - `VERBOSE` opposite of 'QUIET'; will log even debug information
   - `LOGFILE` specifies the file to write logs to. (default standard output)
-  - `LOGFORMAT` specifies either "text" or "json" output format
+  - `LOGFORMAT` specifies either "text", "json" or "logfmt" output format
     (default "text")
 
 All of these variables are optional and will be given the following default

data/lib/resque/scheduler/configuration.rb

@@ -44,7 +44,7 @@ module Resque
         @logfile ||= environment['LOGFILE']
       end
 
-      # Sets whether to log in 'text' or 'json'
+      # Sets whether to log in 'text', 'json' or 'logfmt'
       attr_writer :logformat
 
       def logformat
@@ -65,6 +65,12 @@ module Resque
         @app_name ||= environment['APP_NAME']
       end
 
+      def delayed_requeue_batch_size
+        @delayed_requeue_batch_size ||= \
+          ENV['DELAYED_REQUEUE_BATCH_SIZE'].to_i if environment['DELAYED_REQUEUE_BATCH_SIZE']
+        @delayed_requeue_batch_size ||= 100
+      end
+
       # Amount of time in seconds to sleep between polls of the delayed
       # queue.  Defaults to 5
       attr_writer :poll_sleep_amount

data/lib/resque/scheduler/delaying_extensions.rb

@@ -90,7 +90,7 @@ module Resque
         redis.rpush("delayed:#{timestamp.to_i}", encode(item))
 
         # Store the timestamps at with this item occurs
-        redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
+        redis.sadd("timestamps:#{encode(item)}", ["delayed:#{timestamp.to_i}"])
 
         # Now, add this timestamp to the zsets.  The score and the value are
         # the same since we'll be querying by timestamp, and we don't have
@@ -140,7 +140,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
 
         encoded_item = redis.lpop(key)
-        redis.srem("timestamps:#{encoded_item}", key)
+        redis.srem("timestamps:#{encoded_item}", [key])
         item = decode(encoded_item)
 
         # If the list is empty, remove it.
@@ -257,7 +257,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
         encoded_job = encode(job_to_hash(klass, args))
 
-        redis.srem("timestamps:#{encoded_job}", key)
+        redis.srem("timestamps:#{encoded_job}", [key])
         count = redis.lrem(key, 0, encoded_job)
         clean_up_timestamp(key, timestamp)
 
@@ -297,6 +297,22 @@ module Resque
         redis.hget('delayed:last_enqueued_at', job_name)
       end
 
+      def clean_up_timestamp(key, timestamp)
+        # Use a watch here to ensure nobody adds jobs to this delayed
+        # queue while we're removing it.
+        redis.watch(key) do
+          if redis.llen(key).to_i == 0
+            # If the list is empty, remove it.
+            redis.multi do |transaction|
+              transaction.del(key)
+              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
+            end
+          else
+            redis.redis.unwatch
+          end
+        end
+      end
+
       private
 
       def job_to_hash(klass, args)
@@ -317,7 +333,7 @@ module Resque
         replies = redis.pipelined do |pipeline|
           timestamps.each do |key|
             pipeline.lrem(key, 0, encoded_job)
-            pipeline.srem("timestamps:#{encoded_job}", key)
+            pipeline.srem("timestamps:#{encoded_job}", [key])
           end
         end
 
@@ -328,22 +344,6 @@ module Resque
         replies.each_slice(2).map(&:first).inject(:+)
       end
 
-      def clean_up_timestamp(key, timestamp)
-        # Use a watch here to ensure nobody adds jobs to this delayed
-        # queue while we're removing it.
-        redis.watch(key) do
-          if redis.llen(key).to_i == 0
-            # If the list is empty, remove it.
-            redis.multi do |transaction|
-              transaction.del(key)
-              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
-            end
-          else
-            redis.redis.unwatch
-          end
-        end
-      end
-
       def search_first_delayed_timestamp_in_range(start_at, stop_at)
         start_at = start_at.nil? ? '-inf' : start_at.to_i
         stop_at = stop_at.nil? ? '+inf' : stop_at.to_i

data/lib/resque/scheduler/logger_builder.rb

@@ -15,7 +15,7 @@ module Resque
       #   - :quiet if logger needs to be silent for all levels. Default - false
       #   - :verbose if there is a need in debug messages. Default - false
       #   - :log_dev to output logs into a desired file. Default - STDOUT
-      #   - :format log format, either 'text' or 'json'. Default - 'text'
+      #   - :format log format, either 'text', 'json' or 'logfmt'. Default - 'text'
       #
       # Example:
       #
@@ -32,6 +32,7 @@ module Resque
       # Returns an instance of MonoLogger
       def build
         logger = MonoLogger.new(@log_dev)
+        logger.progname = 'resque-scheduler'.freeze
         logger.level = level
         logger.formatter = send(:"#{@format}_formatter")
         logger
@@ -50,8 +51,8 @@ module Resque
       end
 
       def text_formatter
-        proc do |severity, datetime, _progname, msg|
-          "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
+        proc do |severity, datetime, progname, msg|
+          "#{progname}: [#{severity}] #{datetime.iso8601}: #{msg}\n"
         end
       end
 
@@ -59,7 +60,7 @@ module Resque
         proc do |severity, datetime, progname, msg|
           require 'json'
           JSON.dump(
-            name: 'resque-scheduler',
+            name: progname,
             progname: progname,
             level: severity,
             timestamp: datetime.iso8601,
@@ -67,6 +68,15 @@ module Resque
           ) + "\n"
         end
       end
+
+      def logfmt_formatter
+        proc do |severity, datetime, progname, msg|
+          "timestamp=\"#{datetime.iso8601}\" " \
+          "level=\"#{severity}\" " \
+          "progname=\"#{progname}\" " \
+          "msg=\"#{msg}\"\n"
+        end
+      end
     end
   end
 end

data/lib/resque/scheduler/scheduling_extensions.rb

@@ -91,7 +91,7 @@ module Resque
           non_persistent_schedules[name] = decode(encode(config))
         end
 
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
         reload_schedule! if reload
       end
 
@@ -105,7 +105,7 @@ module Resque
       def remove_schedule(name, reload = true)
         non_persistent_schedules.delete(name)
         redis.hdel(:persistent_schedules, name)
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
 
         reload_schedule! if reload
       end

data/lib/resque/scheduler/server/views/delayed.erb

@@ -16,6 +16,14 @@
   Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%= size %></b> timestamps
 </p>
 
+<% if size > 0 %>
+  <div style="padding-bottom: 10px">
+    <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed confirmSubmission'>
+      <input type='submit' name='' value='Clear Delayed Jobs' />
+    </form>
+  </div>
+<% end %>
+
 <table>
   <tr>
     <th></th>
@@ -27,7 +35,7 @@
   </tr>
   <% resque.delayed_queue_peek(start, 20).each do |timestamp| %>
     <tr>
-      <td>
+      <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
         <form action="<%= u "/delayed/queue_now" %>" method="post">
           <input type="hidden" name="timestamp" value="<%= timestamp.to_i %>">
           <input type="submit" value="Queue now">
@@ -53,11 +61,4 @@
   <% end %>
 </table>
 
-<% if size > 0 %>
-  <br>
-  <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed'>
-    <input type='submit' name='' value='Clear Delayed Jobs' />
-  </form>
-<% end %>
-
 <%= partial :next_more, :start => start, :size => size %>

data/lib/resque/scheduler/server/views/search.erb

@@ -13,13 +13,13 @@
   </tr>
   <% delayed.each do |job| %>
       <tr>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/queue_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="submit" value="Queue now">
           </form>
         </td>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/cancel_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="hidden" name="klass" value="<%= job['class'] %>">

data/lib/resque/scheduler/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Scheduler
-    VERSION = '4.8.0'.freeze
+    VERSION = '4.10.0'.freeze
   end
 end

data/lib/resque/scheduler.rb

@@ -206,16 +206,80 @@ module Resque
 
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
-        item = nil
+        count = 0
+        batch_size = delayed_requeue_batch_size
+        actual_batch_size = nil
+
+        log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
+
         loop do
           handle_shutdown do
             # Continually check that it is still the master
-            item = enqueue_next_item(timestamp) if am_master
+            if am_master
+              actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
+                                                                       batch_size)
+            end
           end
-          # continue processing until there are no more ready items in this
-          # timestamp
-          break if item.nil?
+
+          count += actual_batch_size
+          log "queued #{count} jobs" if actual_batch_size != -1
+
+          # continue processing until there are no more items in this
+          # timestamp. If we don't have a full batch, this is the last one.
+          # This also breaks us in the event of a redis transaction failure
+          # i.e. enqueue_items_in_batch_for_timestamp returned -1
+          break if actual_batch_size < batch_size
         end
+
+        log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
+      end
+
+      def timestamp_key(timestamp)
+        "delayed:#{timestamp.to_i}"
+      end
+
+      def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
+        timestamp_bucket_key = timestamp_key(timestamp)
+
+        encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
+
+        # Watch is used to ensure that the timestamp bucket we are operating on
+        # is not altered by any other clients between the watch call and when we call exec
+        # (to execute the multi block). We should error catch on the redis.exec return value
+        # as that will indicate if the entire transaction was aborted or not. Though we should
+        # be safe as our ltrim is inside the multi block and therefore also would have been
+        # aborted. So nothing would have been queued, but also nothing lost from the bucket.
+        watch_result = Resque.redis.watch(timestamp_bucket_key) do
+          Resque.redis.multi do |pipeline|
+            encoded_jobs_to_requeue.each do |encoded_job|
+              pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
+
+              decoded_job = Resque.decode(encoded_job)
+              enqueue(decoded_job)
+            end
+
+            pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
+          end
+        end
+
+        # Did the multi block successfully remove from this timestamp and enqueue the jobs?
+        success = !watch_result.nil?
+
+        # If this was the last batch in this timestamp bucket, clean up
+        if success && encoded_jobs_to_requeue.count < batch_size
+          Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
+        end
+
+        unless success
+          # Our batched transaction failed in Redis due to the timestamp_bucket_key value
+          # being modified while we built our multi block. We return -1 to ensure we break
+          # out of the loop iterating on this timestamp so it can be re-processed via the
+          # loop in handle_delayed_items.
+          return -1
+        end
+
+        # will return 0 if none were left to batch
+        encoded_jobs_to_requeue.count
       end
 
       def enqueue(config)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-scheduler
 version: !ruby/object:Gem::Version
-  version: 4.8.0
+  version: 4.10.0
 platform: ruby
 authors:
 - Ben VandenBos
@@ -9,10 +9,10 @@ authors:
 - Ryan Biesemeyer
 - Dan Buch
 - Michael Bianco
-autorequire:
+autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-01-27 00:00:00.000000000 Z
+date: 2023-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -307,7 +307,7 @@ licenses:
 - MIT
 metadata:
   rubygems_mfa_required: 'true'
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -322,8 +322,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Light weight job scheduling on top of Resque
 test_files: []