sidekiq-unique-jobs 7.1.19 → 7.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5432e71679519bcfbc3605de7c156e0820cf2c463ee9a5d458c1b59bd7c63fec
-  data.tar.gz: ecd548ff2add944949acc25341d92e2f74aee5256978d32ee74180ba1eacbd66
+  metadata.gz: 2a112ea37ca98a7909abf3b536e5d5f2b147e3a40f7749bf3c7d819c259d069e
+  data.tar.gz: a3f34c72a78723c0dd441d005ae0300662183ddbe24cde430e7714587feccbd3
 SHA512:
-  metadata.gz: 926bde9f4894aba3806723c1d75faffdf8bde86514b37f69b4156ca12b3dd64228d6eb768a03f521d119ebb42f50bcfbe87392d788f54a6e49b233565743dcb4
-  data.tar.gz: 28d14de13123d83b6d948d8453a901efe0ae1667ebc0d6a444e702371579528eccb4ac42c298aa2eb425c23b402c155cc58f553def21cba9c7a6d46757a167e2
+  metadata.gz: 0a6d73f2beade75f9b0261a65ff3be37dfe0fe35c6ff3550635d3b8c634f1ad80c646c36be63c304806ef4f4f471eb0bb56ad641714ed48a53a14ee1d9f1a1aa
+  data.tar.gz: 93f518d9743445c8b844032dbcd8e583a88909b7e3bed9b88807ce3e3e337d2bdf101b8b413e8610705a22aae98747c8fa63549a2b001adc9800903577ad3694

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## [v7.1.19](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.19) (2022-04-09)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.18...v7.1.19)
+
+**Fixed bugs:**
+
+- concurrent-ruby 1.1.10 spikes volume of jobs [\#701](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/701)
+- Reimplement the entire TimerTask as it was [\#702](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/702) ([mhenrixon](https://github.com/mhenrixon))
+
 ## [v7.1.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.18) (2022-04-05)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.17...v7.1.18)

data/README.md

@@ -123,9 +123,11 @@ Before v7, the middleware was configured automatically. Since some people report
 
 *NOTE* if you want to use the reaper you also need to configure the server middleware.
 
-[A full example](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/myapp/config/initializers/sidekiq.rb#L12)
+The following shows how to modify your `config/initializers/sidekiq.rb` file to use the middleware. [Here is a full example.](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/myapp/config/initializers/sidekiq.rb#L12)
 
 ```ruby
+require "sidekiq-unique-jobs"
+
 Sidekiq.configure_server do |config|
   config.redis = { url: ENV["REDIS_URL"], driver: :hiredis }
 

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -110,13 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        timer_task_options = { run_now: true, execution_interval: reaper_interval }
-
-        if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
-          timer_task_options[:timeout_interval] = reaper_timeout
-        end
-
-        timer_task_options
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -133,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -11,6 +11,8 @@ module SidekiqUniqueJobs
     #
     # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
@@ -30,6 +32,16 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as integer
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -37,9 +49,11 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests    = SidekiqUniqueJobs::Digests.new
+        @scheduled  = Redis::SortedSet.new(SCHEDULE)
+        @retried    = Redis::SortedSet.new(RETRY)
+        @start_time = time_source.call
+        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -60,7 +74,7 @@ module SidekiqUniqueJobs
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans # rubocop:disable Metrics/MethodLength
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
         page = 0
         per = reaper_count * 2
         orphans = []
@@ -68,12 +82,14 @@ module SidekiqUniqueJobs
 
         while results.size.positive?
           results.each do |digest|
+            break if timeout?
             next if belongs_to_job?(digest)
 
             orphans << digest
             break if orphans.size >= reaper_count
           end
 
+          break if timeout?
           break if orphans.size >= reaper_count
 
           page += 1
@@ -83,6 +99,14 @@ module SidekiqUniqueJobs
         orphans
       end
 
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_time
+      end
+
       #
       # Checks if the digest has a matching job.
       #   1. It checks the scheduled set

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -26,9 +26,7 @@ module SidekiqUniqueJobs
   # Should the task experience an unrecoverable crash only the task thread will
   # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
   # `TimerTask` thread can respond to the success or failure of the task,
-  # performing logging or ancillary operations. `TimerTask` can also be
-  # configured with a timeout value allowing it to kill a task that runs too
-  # long.
+  # performing logging or ancillary operations.
   #
   # One other advantage of `TimerTask` is that it forces the business logic to
   # be completely decoupled from the concurrency logic. The business logic can
@@ -49,9 +47,7 @@ module SidekiqUniqueJobs
   # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
   # Observable} module. On execution the `TimerTask` will notify the observers
   # with three arguments: time of execution, the result of the block (or nil on
-  # failure), and any raised exceptions (or nil on success). If the timeout
-  # interval is exceeded the observer will receive a `Concurrent::TimeoutError`
-  # object as the third argument.
+  # failure), and any raised exceptions (or nil on success).
   #
   # @!macro copy_options
   #
@@ -60,20 +56,18 @@ module SidekiqUniqueJobs
   #   task.execute
   #
   #   task.execution_interval #=> 60 (default)
-  #   task.timeout_interval   #=> 30 (default)
   #
   #   # wait 60 seconds...
   #   #=> 'Boom!'
   #
   #   task.shutdown #=> true
   #
-  # @example Configuring `:execution_interval` and `:timeout_interval`
-  #   task = Concurrent::TimerTask.new(execution_interval: 5, timeout_interval: 5) do
+  # @example Configuring `:execution_interval`
+  #   task = Concurrent::TimerTask.new(execution_interval: 5) do
   #          puts 'Boom!'
   #        end
   #
   #   task.execution_interval #=> 5
-  #   task.timeout_interval   #=> 5
   #
   # @example Immediate execution with `:run_now`
   #   task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
@@ -116,15 +110,13 @@ module SidekiqUniqueJobs
   #     def update(time, result, ex)
   #       if result
   #         print "(#{time}) Execution successfully returned #{result}\n"
-  #       elsif ex.is_a?(Concurrent::TimeoutError)
-  #         print "(#{time}) Execution timed out\n"
   #       else
   #         print "(#{time}) Execution failed with error #{ex}\n"
   #       end
   #     end
   #   end
   #
-  #   task = Concurrent::TimerTask.new(execution_interval: 1, timeout_interval: 1){ 42 }
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
   #   task.add_observer(TaskObserver.new)
   #   task.execute
   #   sleep 4
@@ -134,7 +126,7 @@ module SidekiqUniqueJobs
   #   #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
   #   task.shutdown
   #
-  #   task = Concurrent::TimerTask.new(execution_interval: 1, timeout_interval: 1){ sleep }
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
   #   task.add_observer(TaskObserver.new)
   #   task.execute
   #
@@ -154,7 +146,7 @@ module SidekiqUniqueJobs
   #
   # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
   # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
-  class TimerTask < Concurrent::RubyExecutorService # rubocop:disable Metrics/ClassLength
+  class TimerTask < Concurrent::RubyExecutorService
     include Concurrent::Concern::Dereferenceable
     include Concurrent::Concern::Observable
 
@@ -170,8 +162,6 @@ module SidekiqUniqueJobs
     #   @param [Hash] opts the options defining task execution.
     #   @option opts [Integer] :execution_interval number of seconds between
     #     task executions (default: EXECUTION_INTERVAL)
-    #   @option opts [Integer] :timeout_interval number of seconds a task can
-    #     run before it is considered to have failed (default: TIMEOUT_INTERVAL)
     #   @option opts [Boolean] :run_now Whether to run the task immediately
     #     upon instantiation or to wait until the first #  execution_interval
     #     has passed (default: false)
@@ -233,7 +223,7 @@ module SidekiqUniqueJobs
     #   task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
     #   task.running? #=> true
     def self.execute(opts = {}, &task)
-      SidekiqUniqueJobs::TimerTask.new(opts, &task).execute
+      TimerTask.new(opts, &task).execute
     end
 
     # @!attribute [rw] execution_interval
@@ -252,22 +242,6 @@ module SidekiqUniqueJobs
       synchronize { @execution_interval = value }
     end
 
-    # @!attribute [rw] timeout_interval
-    # @return [Fixnum] Number of seconds the task can run before it is
-    #   considered to have failed.
-    def timeout_interval
-      synchronize { @timeout_interval }
-    end
-
-    # @!attribute [rw] timeout_interval
-    # @return [Fixnum] Number of seconds the task can run before it is
-    #   considered to have failed.
-    def timeout_interval=(value)
-      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
-
-      synchronize { @timeout_interval = value }
-    end
-
     private :post, :<<
 
     private
@@ -276,12 +250,13 @@ module SidekiqUniqueJobs
       set_deref_options(opts)
 
       self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
-      self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
-      @run_now  = opts[:now] || opts[:run_now]
-      @executor = Concurrent::RubySingleThreadExecutor.new
-      @running  = Concurrent::AtomicBoolean.new(false)
-      @task     = task
-      @value    = nil
+      if opts[:timeout] || opts[:timeout_interval]
+        warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
+      end
+      @run_now = opts[:now] || opts[:run_now]
+      @executor = Concurrent::SafeTaskExecutor.new(task)
+      @running = Concurrent::AtomicBoolean.new(false)
+      @value = nil
 
       self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
     end
@@ -306,52 +281,19 @@ module SidekiqUniqueJobs
     end
 
     # @!visibility private
-    def execute_task(completion) # rubocop:disable Metrics/MethodLength
+    def execute_task(completion)
       return nil unless @running.true?
 
-      timeout_task = -> { timeout_task(completion) }
-
-      Concurrent::ScheduledTask.execute(
-        timeout_interval,
-        args: [completion],
-        &timeout_task
-      )
-      @thread_completed = Concurrent::Event.new
-
-      @value = @reason  = nil
-      @executor.post do
-        @value = @task.call(self)
-      rescue Exception => ex # rubocop:disable Lint/RescueException
-        @reason = ex
-      ensure
-        @thread_completed.set
-      end
-
-      @thread_completed.wait
-
+      _success, value, reason = @executor.execute(self)
       if completion.try?
+        self.value = value
         schedule_next_task
         time = Time.now
         observers.notify_observers do
-          [time, value, @reason]
+          [time, self.value, reason]
         end
       end
       nil
     end
-
-    # @!visibility private
-    def timeout_task(completion)
-      return unless @running.true?
-      return unless completion.try?
-
-      @executor.kill
-      @executor.wait_for_termination
-      @executor = Concurrent::RubySingleThreadExecutor.new
-
-      @thread_completed.set
-
-      schedule_next_task
-      observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
-    end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.19"
+  VERSION = "7.1.20"
 end

data/lib/tasks/changelog.rake

@@ -17,7 +17,7 @@ COMMIT_CHANGELOG_CMD = "git commit -a -m 'Update changelog'"
 desc "Generate a Changelog"
 task :changelog do
   sh("git checkout main")
-  sh(*CHANGELOG_CMD.push(ENV["CHANGELOG_GITHUB_TOKEN"]))
+  sh(*CHANGELOG_CMD.push(ENV.fetch("CHANGELOG_GITHUB_TOKEN", nil)))
   sh(ADD_CHANGELOG_CMD)
   sh(COMMIT_CHANGELOG_CMD)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.19
+  version: 7.1.20
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-09 00:00:00.000000000 Z
+date: 2022-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script