resque-scheduler 4.6.0 → 4.10.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cf266ccd935c9464051349f32169248c7fab52aa84304a98f0d05e5e91ccec22
-  data.tar.gz: 517e84a33953fa3bf695ae26f23912d436fc86b7dc0ac9502fb09ca1cc02f1c3
+  metadata.gz: 6cf8e385c64ab6efafcc8f3a499ec2dc8d7be736604e7d4578dd356f1c19d6e9
+  data.tar.gz: a564652590bf3c5dc85695cbbb57fb4ca85873de78c94f0580a104ef5d98cf55
 SHA512:
-  metadata.gz: 91b8916a16ad2a6a20860001f137fb558d6dbfa7aca3c67abb022acc298a3f2afac7c1527a043b2d5ed3fa582d0203cc988b101cc45819ae3832145328cdea7d
-  data.tar.gz: 90669eb2a75e71612b1505c87a6ed704e70b80b34aa89023ce84e8a256c1d3e0e4eb2a7931c8e480ff99524ed489bb97ed2d98f512f6646f19d50751613ac97c
+  metadata.gz: 2ab0cea69751963f68055eae115c349a885439a5b33c1c3046a5abd4fc30b676233964d430fa5aeb3eb7960d9339772d41ca31c575156eb4e6583cc66de7497e
+  data.tar.gz: f0a4ab726ea640f9df76bbb824078252fb90de9d40c90220611c71069363025933f4a97200c73c13c7c91ff4f02a8c64c735372bbd2e0510e4e5bd021ec081fb

data/.github/workflows/codeql-analysis.yml

@@ -27,7 +27,7 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

data/.github/workflows/rubocop.yml

@@ -14,11 +14,11 @@ jobs:
       matrix:
         os: [ubuntu-latest]
         ruby: [
-            2.4
+            2.7
         ]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: ${{ matrix.ruby }}

data/.github/workflows/ruby.yml

@@ -26,38 +26,53 @@ jobs:
           - 2.7
           - "3.0"
           - 3.1
+          - 3.2
         resque-version:
           - "master"
-          - "~> 2.2.0"
+          - "~> 2.4.0"
           - "~> 1.27"
         rufus-scheduler:
           - "3.2"
           - "3.4"
           - "3.5"
           - "3.6"
+        redis-version:
+          - "~> 4.x"
+          - "~> 5.x"
         exclude:
           - ruby-version: head
             rufus-scheduler: 3.2
+          - ruby-version: 3.2
+            rufus-scheduler: 3.2
 
           - ruby-version: 2.3
-            resque-version: 1.27
+            resque-version: "~> 1.27"
             rufus-scheduler: 3.4
           - ruby-version: 2.3
-            resque-version: 1.27
+            resque-version: "~> 1.27"
             rufus-scheduler: 3.5
           - ruby-version: 2.5
-            resque-version: 2.2.0
+            resque-version: "~> 2.4.0"
             rufus-scheduler: 3.5
           - ruby-version: 2.5
             resque-version: master
             rufus-scheduler: 3.2
+
+          - ruby-version: 2.3
+            redis-version: "~> 5.x"
+          - ruby-version: 2.4
+            redis-version: "~> 5.x"
+
+          - resque-version: "~> 1.27"
+            redis-version: "~> 5.x"
     env:
+      REDIS_VERSION: "${{ matrix.redis-version }}"
       RESQUE: "${{ matrix.resque-version }}"
       RUFUS_SCHEDULER: "${{ matrix.rufus-scheduler }}"
       COVERAGE: 1
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: "${{ matrix.ruby-version }}"

data/AUTHORS.md

@@ -29,6 +29,7 @@ Resque Scheduler authors
 - Henrik Nyh
 - Hormoz Kheradmand
 - Ian Davies
+- Irving Reid
 - James Le Cuirot
 - Jarkko Mönkkönen
 - Jimmy Chao
@@ -87,4 +88,5 @@ Resque Scheduler authors
 - malomalo
 - sawanoboly
 - serek
-- iloveitaly
+- iloveitaly
+- treacher

data/CHANGELOG.md

@@ -2,6 +2,41 @@
 
 **ATTN**: This project uses [semantic versioning](http://semver.org/).
 
+## [4.10.2] - 2023-12-15
+### Fixed
+* Finish fixing CVE-2022-44303, XSS in delayed_schedules by @PatrickTulskie in #783
+
+## [4.10.1] - 2023-12-15
+### Fixed
+* Fix json dump crasher with JSON 2.7.0+ by @PatrickTulskie in #781
+* Fix for #761, reflected XSS in delayed_schedules.erb by @PatrickTulskie in #780
+
+## [4.10.0] - 2023-08-20
+### Added
+* Add logfmt option for logging output (#763)
+
+### Fixed
+* Rubocop Fixes (#771)
+
+## [4.9.0] - 2023-05-31
+### Changed
+- Adding batching to re-queuing for timestamp by @brennen-stripe in #767
+- Use non-deprecated form of Redis.sadd/srem by @irvingreid in #752
+- Prompt for confirmation on 'Clear Delayed Jobs' by @mishina2228 in #754
+- Address some deprecation warnings in the test suite and with srem/lrem pipelined usage by @PatrickTulskie in #770
+
+### Fixed
+- Update CI matrix and fix tests by @mishina2228 in #766
+
+## [4.8.0] - 2023-27-1
+- Replace deprecated Socket.gethostname with Addrinfo.getaddrinfo to fix deprecation warnings (#753)
+
+## [4.7.0] - 2022-10-6
+### Fixed
+- Fix tests for redis-rb 5.0 (#757)
+- Use Resque::DataStore#reconnect for redis-rb 5.0 compat (#757)
+- Test suite runs `flushall` to prevent flakiness (#757)
+
 ## [4.6.0] - 2022-08-04
 ### Changed
 - Remove support for Ruby < 2.3

data/Gemfile

@@ -15,4 +15,15 @@ else
   gem 'rufus-scheduler', rufus_scheduler_version
 end
 
+case redis_version = ENV.fetch('REDIS_VERSION', 'latest')
+when 'master'
+  gem 'redis', git: 'https://github.com/redis/redis-rb'
+when 'latest'
+  gem 'redis'
+else
+  gem 'redis', redis_version
+end
+
+gem 'sinatra', '> 2.0'
+
 gemspec

data/README.md

@@ -139,7 +139,7 @@ requiring `resque` and `resque/scheduler` (default empty).
 * `RESQUE_SCHEDULER_INTERVAL` - Interval in seconds for checking if a
 scheduled job must run (coerced with `Kernel#Float()`) (default `5`)
 * `LOGFILE` - Log file name (default empty, meaning `$stdout`)
-* `LOGFORMAT` - Log output format to use (either `'text'` or `'json'`,
+* `LOGFORMAT` - Log output format to use (either `'text'`, `'json'` or `'logfmt'`,
 default `'text'`)
 * `PIDFILE` - If non-empty, write process PID to file (default empty)
 * `QUIET` - Silence most output if non-empty (equivalent to a level of
@@ -158,7 +158,7 @@ following task to wherever tasks are kept, such as
 ```ruby
 task 'resque:pool:setup' do
   Resque::Pool.after_prefork do |job|
-    Resque.redis._client.reconnect
+    Resque.redis.reconnect
   end
 end
 ```
@@ -663,7 +663,7 @@ are toggled by environment variables:
   - `QUIET` will stop logging anything. Completely silent.
   - `VERBOSE` opposite of 'QUIET'; will log even debug information
   - `LOGFILE` specifies the file to write logs to. (default standard output)
-  - `LOGFORMAT` specifies either "text" or "json" output format
+  - `LOGFORMAT` specifies either "text", "json" or "logfmt" output format
     (default "text")
 
 All of these variables are optional and will be given the following default

data/lib/resque/scheduler/configuration.rb

@@ -44,7 +44,7 @@ module Resque
         @logfile ||= environment['LOGFILE']
       end
 
-      # Sets whether to log in 'text' or 'json'
+      # Sets whether to log in 'text', 'json' or 'logfmt'
       attr_writer :logformat
 
       def logformat
@@ -65,6 +65,12 @@ module Resque
         @app_name ||= environment['APP_NAME']
       end
 
+      def delayed_requeue_batch_size
+        @delayed_requeue_batch_size ||= \
+          ENV['DELAYED_REQUEUE_BATCH_SIZE'].to_i if environment['DELAYED_REQUEUE_BATCH_SIZE']
+        @delayed_requeue_batch_size ||= 100
+      end
+
       # Amount of time in seconds to sleep between polls of the delayed
       # queue.  Defaults to 5
       attr_writer :poll_sleep_amount

data/lib/resque/scheduler/delaying_extensions.rb

@@ -90,7 +90,7 @@ module Resque
         redis.rpush("delayed:#{timestamp.to_i}", encode(item))
 
         # Store the timestamps at with this item occurs
-        redis.sadd("timestamps:#{encode(item)}", "delayed:#{timestamp.to_i}")
+        redis.sadd("timestamps:#{encode(item)}", ["delayed:#{timestamp.to_i}"])
 
         # Now, add this timestamp to the zsets.  The score and the value are
         # the same since we'll be querying by timestamp, and we don't have
@@ -140,7 +140,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
 
         encoded_item = redis.lpop(key)
-        redis.srem("timestamps:#{encoded_item}", key)
+        redis.srem("timestamps:#{encoded_item}", [key])
         item = decode(encoded_item)
 
         # If the list is empty, remove it.
@@ -257,7 +257,7 @@ module Resque
         key = "delayed:#{timestamp.to_i}"
         encoded_job = encode(job_to_hash(klass, args))
 
-        redis.srem("timestamps:#{encoded_job}", key)
+        redis.srem("timestamps:#{encoded_job}", [key])
         count = redis.lrem(key, 0, encoded_job)
         clean_up_timestamp(key, timestamp)
 
@@ -297,6 +297,22 @@ module Resque
         redis.hget('delayed:last_enqueued_at', job_name)
       end
 
+      def clean_up_timestamp(key, timestamp)
+        # Use a watch here to ensure nobody adds jobs to this delayed
+        # queue while we're removing it.
+        redis.watch(key) do
+          if redis.llen(key).to_i == 0
+            # If the list is empty, remove it.
+            redis.multi do |transaction|
+              transaction.del(key)
+              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
+            end
+          else
+            redis.redis.unwatch
+          end
+        end
+      end
+
       private
 
       def job_to_hash(klass, args)
@@ -317,7 +333,7 @@ module Resque
         replies = redis.pipelined do |pipeline|
           timestamps.each do |key|
             pipeline.lrem(key, 0, encoded_job)
-            pipeline.srem("timestamps:#{encoded_job}", key)
+            pipeline.srem("timestamps:#{encoded_job}", [key])
           end
         end
 
@@ -328,22 +344,6 @@ module Resque
         replies.each_slice(2).map(&:first).inject(:+)
       end
 
-      def clean_up_timestamp(key, timestamp)
-        # Use a watch here to ensure nobody adds jobs to this delayed
-        # queue while we're removing it.
-        redis.watch(key) do
-          if redis.llen(key).to_i == 0
-            # If the list is empty, remove it.
-            redis.multi do |transaction|
-              transaction.del(key)
-              transaction.zrem(:delayed_queue_schedule, timestamp.to_i)
-            end
-          else
-            redis.redis.unwatch
-          end
-        end
-      end
-
       def search_first_delayed_timestamp_in_range(start_at, stop_at)
         start_at = start_at.nil? ? '-inf' : start_at.to_i
         stop_at = stop_at.nil? ? '+inf' : stop_at.to_i

data/lib/resque/scheduler/env.rb

@@ -39,7 +39,7 @@ module Resque
                                   end
 
         Process.daemon(true, !Resque::Scheduler.quiet)
-        Resque.redis._client.reconnect
+        Resque.redis.reconnect
       end
 
       def setup_pid_file

data/lib/resque/scheduler/lock/base.rb

@@ -47,7 +47,7 @@ module Resque
 
         def hostname
           local_hostname = Socket.gethostname
-          Socket.gethostbyname(local_hostname).first
+          Addrinfo.getaddrinfo(local_hostname, 'http').first.getnameinfo.first
         rescue
           local_hostname
         end

data/lib/resque/scheduler/logger_builder.rb

@@ -15,7 +15,7 @@ module Resque
       #   - :quiet if logger needs to be silent for all levels. Default - false
       #   - :verbose if there is a need in debug messages. Default - false
       #   - :log_dev to output logs into a desired file. Default - STDOUT
-      #   - :format log format, either 'text' or 'json'. Default - 'text'
+      #   - :format log format, either 'text', 'json' or 'logfmt'. Default - 'text'
       #
       # Example:
       #
@@ -32,6 +32,7 @@ module Resque
       # Returns an instance of MonoLogger
       def build
         logger = MonoLogger.new(@log_dev)
+        logger.progname = 'resque-scheduler'.freeze
         logger.level = level
         logger.formatter = send(:"#{@format}_formatter")
         logger
@@ -50,21 +51,31 @@ module Resque
       end
 
       def text_formatter
-        proc do |severity, datetime, _progname, msg|
-          "resque-scheduler: [#{severity}] #{datetime.iso8601}: #{msg}\n"
+        proc do |severity, datetime, progname, msg|
+          "#{progname}: [#{severity}] #{datetime.iso8601}: #{msg}\n"
         end
       end
 
       def json_formatter
         proc do |severity, datetime, progname, msg|
           require 'json'
-          JSON.dump(
-            name: 'resque-scheduler',
+          log_data = {
+            name: progname,
             progname: progname,
             level: severity,
             timestamp: datetime.iso8601,
             msg: msg
-          ) + "\n"
+          }
+          JSON.dump(log_data) + "\n"
+        end
+      end
+
+      def logfmt_formatter
+        proc do |severity, datetime, progname, msg|
+          "timestamp=\"#{datetime.iso8601}\" " \
+          "level=\"#{severity}\" " \
+          "progname=\"#{progname}\" " \
+          "msg=\"#{msg}\"\n"
         end
       end
     end

data/lib/resque/scheduler/scheduling_extensions.rb

@@ -91,7 +91,7 @@ module Resque
           non_persistent_schedules[name] = decode(encode(config))
         end
 
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
         reload_schedule! if reload
       end
 
@@ -105,7 +105,7 @@ module Resque
       def remove_schedule(name, reload = true)
         non_persistent_schedules.delete(name)
         redis.hdel(:persistent_schedules, name)
-        redis.sadd(:schedules_changed, name)
+        redis.sadd(:schedules_changed, [name])
 
         reload_schedule! if reload
       end

data/lib/resque/scheduler/server/views/delayed.erb

@@ -16,6 +16,14 @@
   Showing <%= start = params[:start].to_i %> to <%= start + 20 %> of <b><%= size %></b> timestamps
 </p>
 
+<% if size > 0 %>
+  <div style="padding-bottom: 10px">
+    <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed confirmSubmission'>
+      <input type='submit' name='' value='Clear Delayed Jobs' />
+    </form>
+  </div>
+<% end %>
+
 <table>
   <tr>
     <th></th>
@@ -27,7 +35,7 @@
   </tr>
   <% resque.delayed_queue_peek(start, 20).each do |timestamp| %>
     <tr>
-      <td>
+      <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
         <form action="<%= u "/delayed/queue_now" %>" method="post">
           <input type="hidden" name="timestamp" value="<%= timestamp.to_i %>">
           <input type="submit" value="Queue now">
@@ -53,11 +61,4 @@
   <% end %>
 </table>
 
-<% if size > 0 %>
-  <br>
-  <form method="POST" action="<%= u 'delayed/clear' %>" class='clear-delayed'>
-    <input type='submit' name='' value='Clear Delayed Jobs' />
-  </form>
-<% end %>
-
 <%= partial :next_more, :start => start, :size => size %>

data/lib/resque/scheduler/server/views/delayed_schedules.erb

@@ -1,4 +1,4 @@
-<h1>Delayed jobs scheduled for <%= params[:klass] %> (<%= show_job_arguments(@args) %>)</h1>
+<h1>Delayed jobs scheduled for <%=h params[:klass] %> (<%=h show_job_arguments(@args) %>)</h1>
 
 <table class='jobs'>
 <tr>

data/lib/resque/scheduler/server/views/search.erb

@@ -13,13 +13,13 @@
   </tr>
   <% delayed.each do |job| %>
       <tr>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/queue_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="submit" value="Queue now">
           </form>
         </td>
-        <td>
+        <td style="padding-top: 12px; padding-bottom: 2px; width: 10px">
           <form action="<%= u "/delayed/cancel_now" %>" method="post">
             <input type="hidden" name="timestamp" value="<%= job['timestamp'].to_i %>">
             <input type="hidden" name="klass" value="<%= job['class'] %>">

data/lib/resque/scheduler/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Scheduler
-    VERSION = '4.6.0'.freeze
+    VERSION = '4.10.2'.freeze
   end
 end

data/lib/resque/scheduler.rb

@@ -206,16 +206,80 @@ module Resque
 
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
-        item = nil
+        count = 0
+        batch_size = delayed_requeue_batch_size
+        actual_batch_size = nil
+
+        log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
+
         loop do
           handle_shutdown do
             # Continually check that it is still the master
-            item = enqueue_next_item(timestamp) if am_master
+            if am_master
+              actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
+                                                                       batch_size)
+            end
           end
-          # continue processing until there are no more ready items in this
-          # timestamp
-          break if item.nil?
+
+          count += actual_batch_size
+          log "queued #{count} jobs" if actual_batch_size != -1
+
+          # continue processing until there are no more items in this
+          # timestamp. If we don't have a full batch, this is the last one.
+          # This also breaks us in the event of a redis transaction failure
+          # i.e. enqueue_items_in_batch_for_timestamp returned -1
+          break if actual_batch_size < batch_size
         end
+
+        log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
+      end
+
+      def timestamp_key(timestamp)
+        "delayed:#{timestamp.to_i}"
+      end
+
+      def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
+        timestamp_bucket_key = timestamp_key(timestamp)
+
+        encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
+
+        # Watch is used to ensure that the timestamp bucket we are operating on
+        # is not altered by any other clients between the watch call and when we call exec
+        # (to execute the multi block). We should error catch on the redis.exec return value
+        # as that will indicate if the entire transaction was aborted or not. Though we should
+        # be safe as our ltrim is inside the multi block and therefore also would have been
+        # aborted. So nothing would have been queued, but also nothing lost from the bucket.
+        watch_result = Resque.redis.watch(timestamp_bucket_key) do
+          Resque.redis.multi do |pipeline|
+            encoded_jobs_to_requeue.each do |encoded_job|
+              pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
+
+              decoded_job = Resque.decode(encoded_job)
+              enqueue(decoded_job)
+            end
+
+            pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
+          end
+        end
+
+        # Did the multi block successfully remove from this timestamp and enqueue the jobs?
+        success = !watch_result.nil?
+
+        # If this was the last batch in this timestamp bucket, clean up
+        if success && encoded_jobs_to_requeue.count < batch_size
+          Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
+        end
+
+        unless success
+          # Our batched transaction failed in Redis due to the timestamp_bucket_key value
+          # being modified while we built our multi block. We return -1 to ensure we break
+          # out of the loop iterating on this timestamp so it can be re-processed via the
+          # loop in handle_delayed_items.
+          return -1
+        end
+
+        # will return 0 if none were left to batch
+        encoded_jobs_to_requeue.count
       end
 
       def enqueue(config)

data/resque-scheduler.gemspec

@@ -12,6 +12,7 @@ Gem::Specification.new do |spec|
     Ryan Biesemeyer
     Dan Buch
     Michael Bianco
+    Patrick Tulskie
   EOF
   spec.email = %w(
     bvandenbos@gmail.com
@@ -19,6 +20,7 @@ Gem::Specification.new do |spec|
     ryan@yaauie.com
     dan@meatballhat.com
     mike@mikebian.co
+    patricktulskie@gmail.com
   )
   spec.summary = 'Light weight job scheduling on top of Resque'
   spec.description = <<-DESCRIPTION

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: resque-scheduler
 version: !ruby/object:Gem::Version
-  version: 4.6.0
+  version: 4.10.2
 platform: ruby
 authors:
 - Ben VandenBos
@@ -9,10 +9,11 @@ authors:
 - Ryan Biesemeyer
 - Dan Buch
 - Michael Bianco
+- Patrick Tulskie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-08-20 00:00:00.000000000 Z
+date: 2023-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -254,6 +255,7 @@ email:
 - ryan@yaauie.com
 - dan@meatballhat.com
 - mike@mikebian.co
+- patricktulskie@gmail.com
 executables:
 - resque-scheduler
 extensions: []
@@ -322,7 +324,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Light weight job scheduling on top of Resque