resque-scheduler 4.3.1 → 4.10.2

data/lib/resque/scheduler/util.rb

@@ -4,7 +4,7 @@ module Resque
   module Scheduler
     class Util
       # In order to upgrade to resque(1.25) which has deprecated following
-      # methods, we just added these usefull helpers back to use in Resque
+      # methods, we just added these useful helpers back to use in Resque
       # Scheduler.  refer to:
       # https://github.com/resque/resque-scheduler/pull/273
 

data/lib/resque/scheduler/version.rb

@@ -2,6 +2,6 @@
 
 module Resque
   module Scheduler
-    VERSION = '4.3.1'.freeze
+    VERSION = '4.10.2'.freeze
   end
 end

data/lib/resque/scheduler.rb

@@ -1,5 +1,6 @@
 # vim:fileencoding=utf-8
 
+require 'redis/errors'
 require 'rufus/scheduler'
 require_relative 'scheduler/configuration'
 require_relative 'scheduler/locking'
@@ -13,6 +14,9 @@ module Resque
     autoload :Extension, 'resque/scheduler/extension'
     autoload :Util, 'resque/scheduler/util'
     autoload :VERSION, 'resque/scheduler/version'
+    INTERMITTENT_ERRORS = [
+      Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError, Redis::TimeoutError
+    ].freeze
 
     private
 
@@ -44,13 +48,7 @@ module Resque
         $stdout.sync = true
         $stderr.sync = true
 
-        # Load the schedule into rufus
-        # If dynamic is set, load that schedule otherwise use normal load
-        if dynamic
-          reload_schedule!
-        else
-          load_schedule!
-        end
+        was_master = nil
 
         begin
           @th = Thread.current
@@ -58,11 +56,21 @@ module Resque
           # Now start the scheduling part of the loop.
           loop do
             begin
-              if master?
+              # Check on changes to master/child
+              @am_master = master?
+              if am_master != was_master
+                procline am_master ? 'Master scheduler' : 'Child scheduler'
+
+                # Load schedule because changed
+                reload_schedule!
+              end
+
+              if am_master
                 handle_delayed_items
                 update_schedule if dynamic
               end
-            rescue Errno::EAGAIN, Errno::ECONNRESET, Redis::CannotConnectError => e
+              was_master = am_master
+            rescue *INTERMITTENT_ERRORS => e
               log! e.message
               release_master_lock
             end
@@ -99,7 +107,7 @@ module Resque
         Resque.schedule.each do |name, config|
           load_schedule_job(name, config)
         end
-        Resque.redis.del(:schedules_changed)
+        Resque.redis.del(:schedules_changed) if am_master && dynamic
         procline 'Schedules Loaded'
       end
 
@@ -198,16 +206,80 @@ module Resque
 
       # Enqueues all delayed jobs for a timestamp
       def enqueue_delayed_items_for_timestamp(timestamp)
-        item = nil
+        count = 0
+        batch_size = delayed_requeue_batch_size
+        actual_batch_size = nil
+
+        log "Processing delayed items for timestamp #{timestamp}, in batches of #{batch_size}"
+
         loop do
           handle_shutdown do
             # Continually check that it is still the master
-            item = enqueue_next_item(timestamp) if master?
+            if am_master
+              actual_batch_size = enqueue_items_in_batch_for_timestamp(timestamp,
+                                                                       batch_size)
+            end
+          end
+
+          count += actual_batch_size
+          log "queued #{count} jobs" if actual_batch_size != -1
+
+          # continue processing until there are no more items in this
+          # timestamp. If we don't have a full batch, this is the last one.
+          # This also breaks us in the event of a redis transaction failure
+          # i.e. enqueue_items_in_batch_for_timestamp returned -1
+          break if actual_batch_size < batch_size
+        end
+
+        log "finished queueing #{count} total jobs for timestamp #{timestamp}" if count != -1
+      end
+
+      def timestamp_key(timestamp)
+        "delayed:#{timestamp.to_i}"
+      end
+
+      def enqueue_items_in_batch_for_timestamp(timestamp, batch_size)
+        timestamp_bucket_key = timestamp_key(timestamp)
+
+        encoded_jobs_to_requeue = Resque.redis.lrange(timestamp_bucket_key, 0, batch_size - 1)
+
+        # Watch is used to ensure that the timestamp bucket we are operating on
+        # is not altered by any other clients between the watch call and when we call exec
+        # (to execute the multi block). We should error catch on the redis.exec return value
+        # as that will indicate if the entire transaction was aborted or not. Though we should
+        # be safe as our ltrim is inside the multi block and therefore also would have been
+        # aborted. So nothing would have been queued, but also nothing lost from the bucket.
+        watch_result = Resque.redis.watch(timestamp_bucket_key) do
+          Resque.redis.multi do |pipeline|
+            encoded_jobs_to_requeue.each do |encoded_job|
+              pipeline.srem("timestamps:#{encoded_job}", timestamp_bucket_key)
+
+              decoded_job = Resque.decode(encoded_job)
+              enqueue(decoded_job)
+            end
+
+            pipeline.ltrim(timestamp_bucket_key, batch_size, -1)
           end
-          # continue processing until there are no more ready items in this
-          # timestamp
-          break if item.nil?
         end
+
+        # Did the multi block successfully remove from this timestamp and enqueue the jobs?
+        success = !watch_result.nil?
+
+        # If this was the last batch in this timestamp bucket, clean up
+        if success && encoded_jobs_to_requeue.count < batch_size
+          Resque.clean_up_timestamp(timestamp_bucket_key, timestamp)
+        end
+
+        unless success
+          # Our batched transaction failed in Redis due to the timestamp_bucket_key value
+          # being modified while we built our multi block. We return -1 to ensure we break
+          # out of the loop iterating on this timestamp so it can be re-processed via the
+          # loop in handle_delayed_items.
+          return -1
+        end
+
+        # will return 0 if none were left to batch
+        encoded_jobs_to_requeue.count
       end
 
       def enqueue(config)
@@ -243,7 +315,7 @@ module Resque
         if job_klass && job_klass != 'Resque::Job'
           # The custom job class API must offer a static "scheduled" method. If
           # the custom job class can not be constantized (via a requeue call
-          # from the web perhaps), fall back to enqueing normally via
+          # from the web perhaps), fall back to enqueuing normally via
           # Resque::Job.create.
           begin
             Resque::Scheduler::Util.constantize(job_klass).scheduled(
@@ -369,7 +441,6 @@ module Resque
 
       def stop_rufus_scheduler
         rufus_scheduler.shutdown(:wait)
-        rufus_scheduler.join
       end
 
       def before_shutdown
@@ -420,10 +491,10 @@ module Resque
       private
 
       def enqueue_recurring(name, config)
-        if master?
+        if am_master
           log! "queueing #{config['class']} (#{name})"
-          Resque.last_enqueued_at(name, Time.now.to_s)
           enqueue(config)
+          Resque.last_enqueued_at(name, Time.now.to_s)
         end
       end
 
@@ -442,6 +513,11 @@ module Resque
       def internal_name
         "resque-scheduler-#{Resque::Scheduler::VERSION}"
       end
+
+      def am_master
+        @am_master = master? unless defined?(@am_master)
+        @am_master
+      end
     end
   end
 end

data/resque-scheduler.gemspec

@@ -11,12 +11,16 @@ Gem::Specification.new do |spec|
     Simon Eskildsen
     Ryan Biesemeyer
     Dan Buch
+    Michael Bianco
+    Patrick Tulskie
   EOF
   spec.email = %w(
     bvandenbos@gmail.com
     sirup@sirupsen.com
     ryan@yaauie.com
     dan@meatballhat.com
+    mike@mikebian.co
+    patricktulskie@gmail.com
   )
   spec.summary = 'Light weight job scheduling on top of Resque'
   spec.description = <<-DESCRIPTION
@@ -24,13 +28,16 @@ Gem::Specification.new do |spec|
     Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
     Also supports queueing jobs on a fixed, cron-like schedule.
   DESCRIPTION
-  spec.homepage = 'http://github.com/resque/resque-scheduler'
+  spec.homepage = 'https://github.com/resque/resque-scheduler'
   spec.license = 'MIT'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.required_ruby_version = '>= 2.3.0'
 
   spec.files = `git ls-files -z`.split("\0").reject do |f|
     f.match(%r{^(test|spec|features|examples|bin|tasks)/}) ||
-      f.match(/^(Vagrantfile|Gemfile\.lock|appveyor\.yml)/) ||
-      f.match(/^\.(rubocop|simplecov|travis|vagrant|gitignore)/)
+      f.match(/^(Vagrantfile|Gemfile\.lock)/) ||
+      f.match(/^\.(rubocop|simplecov|vagrant|gitignore)/)
   end
   spec.bindir = 'exe'
   spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
@@ -38,7 +45,6 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'bundler'
   spec.add_development_dependency 'json'
-  spec.add_development_dependency 'kramdown'
   spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'mocha'
   spec.add_development_dependency 'pry'
@@ -47,14 +53,16 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'test-unit'
   spec.add_development_dependency 'yard'
-  spec.add_development_dependency 'tzinfo-data'
+  spec.add_development_dependency 'timecop'
 
   # We pin rubocop because new cops have a tendency to result in false-y
   # positives for new contributors, which is not a nice experience.
   spec.add_development_dependency 'rubocop', '~> 0.40.0'
 
   spec.add_runtime_dependency 'mono_logger', '~> 1.0'
-  spec.add_runtime_dependency 'redis', '>= 3.3', '< 5'
-  spec.add_runtime_dependency 'resque', '~> 1.26'
-  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2'
+  spec.add_runtime_dependency 'redis', '>= 3.3'
+  spec.add_runtime_dependency 'resque', '>= 1.27'
+  # rufus-scheduler v3.7 causes a failure in test/multi_process_test.rb
+  # rufus-scheduler v3.3 is missing a to_local method which fails tests
+  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.2', '!= 3.3'
 end

metadata

@@ -1,17 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: resque-scheduler
 version: !ruby/object:Gem::Version
-  version: 4.3.1
+  version: 4.10.2
 platform: ruby
 authors:
 - Ben VandenBos
 - Simon Eskildsen
 - Ryan Biesemeyer
 - Dan Buch
+- Michael Bianco
+- Patrick Tulskie
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-11-20 00:00:00.000000000 Z
+date: 2023-12-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -41,20 +43,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: kramdown
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -168,7 +156,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: tzinfo-data
+  name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -216,9 +204,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -226,23 +211,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '3.3'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: '5'
 - !ruby/object:Gem::Dependency
   name: resque
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.27'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.26'
+        version: '1.27'
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
@@ -250,6 +232,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '3.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -257,6 +242,9 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.2'
+    - - "!="
+      - !ruby/object:Gem::Version
+        version: '3.3'
 description: |2
       Light weight job scheduling on top of Resque.
       Adds methods enqueue_at/enqueue_in to schedule jobs in the future.
@@ -266,11 +254,18 @@ email:
 - sirup@sirupsen.com
 - ryan@yaauie.com
 - dan@meatballhat.com
+- mike@mikebian.co
+- patricktulskie@gmail.com
 executables:
 - resque-scheduler
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/funding.yml"
+- ".github/workflows/codeql-analysis.yml"
+- ".github/workflows/rubocop.yml"
+- ".github/workflows/ruby.yml"
 - AUTHORS.md
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
@@ -309,10 +304,11 @@ files:
 - lib/resque/scheduler/util.rb
 - lib/resque/scheduler/version.rb
 - resque-scheduler.gemspec
-homepage: http://github.com/resque/resque-scheduler
+homepage: https://github.com/resque/resque-scheduler
 licenses:
 - MIT
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -321,15 +317,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 3.0.3.1
 signing_key: 
 specification_version: 4
 summary: Light weight job scheduling on top of Resque