resque-scheduler 4.4.0 → 4.11.0

data/lib/resque/scheduler/signal_handling.rb

@@ -10,13 +10,13 @@ module Resque
       end
 
       # For all signals, set the shutdown flag and wait for current
-      # poll/enqueing to finish (should be almost instant).  In the
+      # poll/enqueuing to finish (should be almost instant).  In the
       # case of sleeping, exit immediately.
       def register_signal_handlers
         (Signal.list.keys & %w(INT TERM USR1 USR2 QUIT)).each do |sig|
           trap(sig) do
             signal_queue << sig
-            # break sleep in the primary scheduler thread, alowing
+            # break sleep in the primary scheduler thread, allowing
             # the signal queue to get processed as soon as possible.
             @th.wakeup if @th && @th.alive?
           end

data/lib/resque/scheduler/util.rb

@@ -4,7 +4,7 @@ module Resque
   module Scheduler
     class Util
       # In order to upgrade to resque(1.25) which has deprecated following
-      # methods, we just added these usefull helpers back to use in Resque
+      # methods, we just added these useful helpers back to use in Resque
       # Scheduler.  refer to:
       # https://github.com/resque/resque-scheduler/pull/273
 

data/lib/resque/scheduler/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Scheduler
-    VERSION = '4.4.0'.freeze
+    VERSION = '4.11.0'.freeze
   end
 end

data/lib/resque/scheduler.rb

@@ -1,5 +1,6 @@
 # vim:fileencoding=utf-8
 
+require 'redis/errors'
 require 'rufus/scheduler'
 require_relative 'scheduler/configuration'
 require_relative 'scheduler/locking'
@@ -205,16 +206,80 @@ module Resque
 
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
-        item = nil
+        count = 0
+        batch_size = delayed_requeue_batch_size
+        actual_batch_size = nil
+
+        log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
+
         loop do
           handle_shutdown do
             # Continually check that it is still the master
-            item = enqueue_next_item(timestamp) if am_master
+            if am_master
+              actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
+                                                                       batch_size)
+            end
           end
-          # continue processing until there are no more ready items in this
-          # timestamp
-          break if item.nil?
+
+          count += actual_batch_size
+          log "queued #{count} jobs" if actual_batch_size != -1
+
+          # continue processing until there are no more items in this
+          # timestamp. If we don't have a full batch, this is the last one.
+          # This also breaks us in the event of a redis transaction failure
+          # i.e. enqueue_items_in_batch_for_timestamp returned -1
+          break if actual_batch_size < batch_size
         end
+
+        log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
+      end
+
+      def timestamp_key(timestamp)
+        "delayed:#{timestamp.to_i}"
+      end
+
+      def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
+        timestamp_bucket_key = timestamp_key(timestamp)
+
+        encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
+
+        # Watch is used to ensure that the timestamp bucket we are operating on
+        # is not altered by any other clients between the watch call and when we call exec
+        # (to execute the multi block). We should error catch on the redis.exec return value
+        # as that will indicate if the entire transaction was aborted or not. Though we should
+        # be safe as our ltrim is inside the multi block and therefore also would have been
+        # aborted. So nothing would have been queued, but also nothing lost from the bucket.
+        watch_result = Resque.redis.watch(timestamp_bucket_key) do
+          Resque.redis.multi do |pipeline|
+            encoded_jobs_to_requeue.each do |encoded_job|
+              pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
+
+              decoded_job = Resque.decode(encoded_job)
+              enqueue(decoded_job)
+            end
+
+            pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
+          end
+        end
+
+        # Did the multi block successfully remove from this timestamp and enqueue the jobs?
+        success = !watch_result.nil?
+
+        # If this was the last batch in this timestamp bucket, clean up
+        if success && encoded_jobs_to_requeue.count < batch_size
+          Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
+        end
+
+        unless success
+          # Our batched transaction failed in Redis due to the timestamp_bucket_key value
+          # being modified while we built our multi block. We return -1 to ensure we break
+          # out of the loop iterating on this timestamp so it can be re-processed via the
+          # loop in handle_delayed_items.
+          return -1
+        end
+
+        # will return 0 if none were left to batch
+        encoded_jobs_to_requeue.count
       end
 
       def enqueue(config)
@@ -250,7 +315,7 @@ module Resque
         if job_klass && job_klass != 'Resque::Job'
           # The custom job class API must offer a static "scheduled" method. If
           # the custom job class can not be constantized (via a requeue call
-          # from the web perhaps), fall back to enqueing normally via
+          # from the web perhaps), fall back to enqueuing normally via
           # Resque::Job.create.
           begin
             Resque::Scheduler::Util.constantize(job_klass).scheduled(
@@ -266,17 +331,16 @@ module Resque
           # for non-existent classes (for example: running scheduler in
           # one app that schedules for another.
           if Class === klass
-            Resque::Scheduler::Plugin.run_before_delayed_enqueue_hooks(
-              klass, *params
-            )
-
-            # If the class is a custom job class, call self#scheduled on it.
-            # This allows you to do things like Resque.enqueue_at(timestamp,
-            # CustomJobClass). Otherwise, pass off to Resque.
-            if klass.respond_to?(:scheduled)
-              klass.scheduled(queue, klass_name, *params)
-            else
-              Resque.enqueue_to(queue, klass, *params)
+            Resque::Scheduler::Plugin.process_schedule_hooks(klass, *params) do
+              Resque::Scheduler::Plugin.run_before_delayed_enqueue_hooks(klass, *params)
+              # If the class is a custom job class, call self#scheduled on it.
+              # This allows you to do things like Resque.enqueue_at(timestamp,
+              # CustomJobClass). Otherwise, pass off to Resque.
+              if klass.respond_to?(:scheduled)
+                klass.scheduled(queue, klass_name, *params)
+              else
+                Resque.enqueue_to(queue, klass, *params)
+              end
             end
           else
             # This will not run the before_hooks in rescue, but will at least
@@ -376,7 +440,6 @@ module Resque
 
       def stop_rufus_scheduler
         rufus_scheduler.shutdown(:wait)
-        rufus_scheduler.join
       end
 
       def before_shutdown

data/resque-scheduler.gemspec

@@ -11,12 +11,16 @@ Gem::Specification.new do |spec|
     Simon Eskildsen
     Ryan Biesemeyer
     Dan Buch
+    Michael Bianco
+    Patrick Tulskie
   EOF
   spec.email = %w(
     bvandenbos@gmail.com
     sirup@sirupsen.com
     ryan@yaauie.com
     dan@meatballhat.com
+    mike@mikebian.co
+    patricktulskie@gmail.com
   )
   spec.summary = 'Light weight job scheduling on top of Resque'
   spec.description = <<-DESCRIPTION
@@ -24,13 +28,16 @@ Gem::Specification.new do |spec|
     Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
     Also supports queueing jobs on a fixed, cron-like schedule.
   DESCRIPTION
-  spec.homepage = 'http://github.com/resque/resque-scheduler'
+  spec.homepage = 'https://github.com/resque/resque-scheduler'
   spec.license = 'MIT'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.required_ruby_version = '>= 2.3.0'
 
   spec.files = `git ls-files -z`.split("\0").reject do |f|
     f.match(%r{^(test|spec|features|examples|bin|tasks)/}) ||
-      f.match(/^(Vagrantfile|Gemfile\.lock|appveyor\.yml)/) ||
-      f.match(/^\.(rubocop|simplecov|travis|vagrant|gitignore)/)
+      f.match(/^(Vagrantfile|Gemfile\.lock)/) ||
+      f.match(/^\.(rubocop|simplecov|vagrant|gitignore)/)
   end
   spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -38,7 +45,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'json'
-  spec.add_development_dependency 'kramdown'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
@@ -47,7 +53,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'yard'
-  spec.add_development_dependency 'tzinfo-data'
+  spec.add_development_dependency 'timecop'
 
   # We pin rubocop because new cops have a tendency to result in false-y
   # positives for new contributors, which is not a nice experience.
@@ -55,6 +61,8 @@ Gem::Specification.new do |spec|
 
   spec.add_runtime_dependency 'mono_logger', '~> 1.0'
   spec.add_runtime_dependency 'redis', '>= 3.3'
-  spec.add_runtime_dependency 'resque', '>= 1.26'
-  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2'
+  spec.add_runtime_dependency 'resque', '>= 1.27'
+  # rufus-scheduler v3.7 causes a failure in test/multi_process_test.rb
+  # rufus-scheduler v3.3 is missing a to_local method which fails tests
+  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2', '!= 3.3'
 end

metadata

@@ -1,17 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: resque-scheduler
 version: !ruby/object:Gem::Version
-  version: 4.4.0
+  version: 4.11.0
 platform: ruby
 authors:
 - Ben VandenBos
 - Simon Eskildsen
 - Ryan Biesemeyer
 - Dan Buch
+- Michael Bianco
+- Patrick Tulskie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-04-15 00:00:00.000000000 Z
+date: 2024-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -41,20 +43,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: kramdown
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -168,7 +156,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tzinfo-data
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -229,14 +217,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.27'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.27'
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -244,6 +232,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '3.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -251,6 +242,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: |2
       Light weight job scheduling on top of Resque.
       Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
@@ -260,11 +254,18 @@ email:
 - sirup@sirupsen.com
 - ryan@yaauie.com
 - dan@meatballhat.com
+- mike@mikebian.co
+- patricktulskie@gmail.com
 executables:
 - resque-scheduler
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/funding.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/rubocop.yml"
+- ".github/workflows/ruby.yml"
 - AUTHORS.md
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -303,10 +304,11 @@ files:
 - lib/resque/scheduler/util.rb
 - lib/resque/scheduler/version.rb
 - resque-scheduler.gemspec
-homepage: http://github.com/resque/resque-scheduler
+homepage: https://github.com/resque/resque-scheduler
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -315,14 +317,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Light weight job scheduling on top of Resque