sidekiq-unique-jobs 7.1.16 → 7.1.19

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4f0f434849fc23d828254f80ab5c7adeaf0a9bbea684071fd562e0c639fa2ef9
-  data.tar.gz: 9ac073e7731f2add9f55ecfce31f6a567723026ab3a97b6d4e31c67127839fa8
+  metadata.gz: 5432e71679519bcfbc3605de7c156e0820cf2c463ee9a5d458c1b59bd7c63fec
+  data.tar.gz: ecd548ff2add944949acc25341d92e2f74aee5256978d32ee74180ba1eacbd66
 SHA512:
-  metadata.gz: 5ed9e4982703780b97574d29f239f29c8a7d4c544203216a10bd52985381a09934adc0d750178760296a8b19d65811929b4a85cc6d1dd3c772291b7338a891d6
-  data.tar.gz: 6f874ec9fbffe76683dfa2009ef1be2b460288af4327a4feaffbb5588848b94f10faf73a1e9d916f5b1b4358af47b28ce3927c06ec81291f4002824e5bf5a37d
+  metadata.gz: 926bde9f4894aba3806723c1d75faffdf8bde86514b37f69b4156ca12b3dd64228d6eb768a03f521d119ebb42f50bcfbe87392d788f54a6e49b233565743dcb4
+  data.tar.gz: 28d14de13123d83b6d948d8453a901efe0ae1667ebc0d6a444e702371579528eccb4ac42c298aa2eb425c23b402c155cc58f553def21cba9c7a6d46757a167e2

data/CHANGELOG.md

@@ -1,5 +1,47 @@
 # Changelog
 
+## [v7.1.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.18) (2022-04-05)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.17...v7.1.18)
+
+**Implemented enhancements:**
+
+- Make sure we reflect on execution failure [\#700](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/700) ([mhenrixon](https://github.com/mhenrixon))
+
+## [v7.1.17](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.17) (2022-04-05)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.16...v7.1.17)
+
+## [v7.1.16](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.16) (2022-04-02)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.15...v7.1.16)
+
+**Implemented enhancements:**
+
+- Abort Ruby Reaper when sidekiq queues are full [\#690](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/690) ([francesmcmullin](https://github.com/francesmcmullin))
+- Quote '3.0' to ensure CI uses Ruby 3.0.x for the 3.0 entry [\#689](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/689) ([petergoldstein](https://github.com/petergoldstein))
+
+**Fixed bugs:**
+
+- Hotfix: Ensure consistent lock args [\#699](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/699) ([mhenrixon](https://github.com/mhenrixon))
+- Expire older changelog entries first [\#698](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/698) ([mhenrixon](https://github.com/mhenrixon))
+- Fix drift [\#688](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/688) ([mhenrixon](https://github.com/mhenrixon))
+
+**Closed issues:**
+
+- concurrent-ruby has dropped support for TimerTask timeouts [\#697](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/697)
+- Most recent changelogs are removed first [\#696](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/696)
+- Improve README slightly [\#694](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/694)
+- locksmith.rb:327: NoMethodError:  undefined method `+' for nil:NilClass [\#686](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/686)
+- lock\_timeout cannot be nil [\#675](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/675)
+- Skip reaping when queues are too large [\#670](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/670)
+
+**Merged pull requests:**
+
+- Improve readme [\#695](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/695) ([fwolfst](https://github.com/fwolfst))
+- Add funding\_uri to gemspec [\#693](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/693) ([fwolfst](https://github.com/fwolfst))
+- Fix worker validator [\#685](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/685) ([victorfgs](https://github.com/victorfgs))
+
 ## [v7.1.15](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.15) (2022-02-10)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.14...v7.1.15)

data/lib/sidekiq_unique_jobs/lock/until_executed.rb

@@ -33,10 +33,14 @@ module SidekiqUniqueJobs
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute
-        locksmith.execute do
+        executed = locksmith.execute do
           yield
           unlock_and_callback
         end
+
+        reflect(:execution_failed, item) unless executed
+
+        nil
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/until_expired.rb

@@ -33,7 +33,9 @@ module SidekiqUniqueJobs
       # Executes in the Sidekiq server process
       # @yield to the worker class perform method
       def execute(&block)
-        locksmith.execute(&block)
+        executed = locksmith.execute(&block)
+
+        reflect(:execution_failed, item) unless executed
       end
     end
   end

data/lib/sidekiq_unique_jobs/lock/while_executing.rb

@@ -47,7 +47,10 @@ module SidekiqUniqueJobs
             locksmith.unlock
           end
 
-          call_strategy(origin: :server, &block) unless executed
+          unless executed
+            reflect(:execution_failed, item)
+            call_strategy(origin: :server, &block)
+          end
         end
       end
 

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -1,84 +1,357 @@
 # frozen_string_literal: true
 
-require "concurrent/version"
-require_relative "version_check"
+require "concurrent/collection/copy_on_notify_observer_set"
+require "concurrent/concern/dereferenceable"
+require "concurrent/concern/observable"
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/executor/executor_service"
+require "concurrent/executor/ruby_executor_service"
+require "concurrent/executor/safe_task_executor"
+require "concurrent/scheduled_task"
 
 module SidekiqUniqueJobs
-  # @see [Concurrent::TimerTask] https://www.rubydoc.info/gems/concurrent-ruby/Concurrent/TimerTask
+  # A very common concurrency pattern is to run a thread that performs a task at
+  # regular intervals. The thread that performs the task sleeps for the given
+  # interval then wakes up and performs the task. Lather, rinse, repeat... This
+  # pattern causes two problems. First, it is difficult to test the business
+  # logic of the task because the task itself is tightly coupled with the
+  # concurrency logic. Second, an exception raised while performing the task can
+  # cause the entire thread to abend. In a long-running application where the
+  # task thread is intended to run for days/weeks/years a crashed task thread
+  # can pose a significant problem. `TimerTask` alleviates both problems.
   #
-  class TimerTask < ::Concurrent::TimerTask
-    if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
+  # When a `TimerTask` is launched it starts a thread for monitoring the
+  # execution interval. The `TimerTask` thread does not perform the task,
+  # however. Instead, the TimerTask launches the task on a separate thread.
+  # Should the task experience an unrecoverable crash only the task thread will
+  # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
+  # `TimerTask` thread can respond to the success or failure of the task,
+  # performing logging or ancillary operations. `TimerTask` can also be
+  # configured with a timeout value allowing it to kill a task that runs too
+  # long.
+  #
+  # One other advantage of `TimerTask` is that it forces the business logic to
+  # be completely decoupled from the concurrency logic. The business logic can
+  # be tested separately then passed to the `TimerTask` for scheduling and
+  # running.
+  #
+  # In some cases it may be necessary for a `TimerTask` to affect its own
+  # execution cycle. To facilitate this, a reference to the TimerTask instance
+  # is passed as an argument to the provided block every time the task is
+  # executed.
+  #
+  # The `TimerTask` class includes the `Dereferenceable` mixin module so the
+  # result of the last execution is always available via the `#value` method.
+  # Dereferencing options can be passed to the `TimerTask` during construction or
+  # at any later time using the `#set_deref_options` method.
+  #
+  # `TimerTask` supports notification through the Ruby standard library
+  # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # Observable} module. On execution the `TimerTask` will notify the observers
+  # with three arguments: time of execution, the result of the block (or nil on
+  # failure), and any raised exceptions (or nil on success). If the timeout
+  # interval is exceeded the observer will receive a `Concurrent::TimeoutError`
+  # object as the third argument.
+  #
+  # @!macro copy_options
+  #
+  # @example Basic usage
+  #   task = Concurrent::TimerTask.new{ puts 'Boom!' }
+  #   task.execute
+  #
+  #   task.execution_interval #=> 60 (default)
+  #   task.timeout_interval   #=> 30 (default)
+  #
+  #   # wait 60 seconds...
+  #   #=> 'Boom!'
+  #
+  #   task.shutdown #=> true
+  #
+  # @example Configuring `:execution_interval` and `:timeout_interval`
+  #   task = Concurrent::TimerTask.new(execution_interval: 5, timeout_interval: 5) do
+  #          puts 'Boom!'
+  #        end
+  #
+  #   task.execution_interval #=> 5
+  #   task.timeout_interval   #=> 5
+  #
+  # @example Immediate execution with `:run_now`
+  #   task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
+  #   task.execute
+  #
+  #   #=> 'Boom!'
+  #
+  # @example Last `#value` and `Dereferenceable` mixin
+  #   task = Concurrent::TimerTask.new(
+  #     dup_on_deref: true,
+  #     execution_interval: 5
+  #   ){ Time.now }
+  #
+  #   task.execute
+  #   Time.now   #=> 2013-11-07 18:06:50 -0500
+  #   sleep(10)
+  #   task.value #=> 2013-11-07 18:06:55 -0500
+  #
+  # @example Controlling execution from within the block
+  #   timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
+  #     task.execution_interval.times{ print 'Boom! ' }
+  #     print "\n"
+  #     task.execution_interval += 1
+  #     if task.execution_interval > 5
+  #       puts 'Stopping...'
+  #       task.shutdown
+  #     end
+  #   end
+  #
+  #   timer_task.execute # blocking call - this task will stop itself
+  #   #=> Boom!
+  #   #=> Boom! Boom!
+  #   #=> Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom! Boom!
+  #   #=> Stopping...
+  #
+  # @example Observation
+  #   class TaskObserver
+  #     def update(time, result, ex)
+  #       if result
+  #         print "(#{time}) Execution successfully returned #{result}\n"
+  #       elsif ex.is_a?(Concurrent::TimeoutError)
+  #         print "(#{time}) Execution timed out\n"
+  #       else
+  #         print "(#{time}) Execution failed with error #{ex}\n"
+  #       end
+  #     end
+  #   end
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1, timeout_interval: 1){ 42 }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #   sleep 4
+  #
+  #   #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1, timeout_interval: 1){ sleep }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:07:25 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:27 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:29 -0400) Execution timed out
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
+  #   task.shutdown
+  #
+  # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
+  class TimerTask < Concurrent::RubyExecutorService # rubocop:disable Metrics/ClassLength
+    include Concurrent::Concern::Dereferenceable
+    include Concurrent::Concern::Observable
 
-      private
+    # Default `:execution_interval` in seconds.
+    EXECUTION_INTERVAL = 60
 
-      def ns_initialize(opts, &task)
-        set_deref_options(opts)
+    # Default `:timeout_interval` in seconds.
+    TIMEOUT_INTERVAL = 30
 
-        self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
-        self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
-        @run_now  = opts[:now] || opts[:run_now]
-        @executor = Concurrent::RubySingleThreadExecutor.new
-        @running  = Concurrent::AtomicBoolean.new(false)
-        @task     = task
-        @value    = nil
+    # Create a new TimerTask with the given task and configuration.
+    #
+    # @!macro timer_task_initialize
+    #   @param [Hash] opts the options defining task execution.
+    #   @option opts [Integer] :execution_interval number of seconds between
+    #     task executions (default: EXECUTION_INTERVAL)
+    #   @option opts [Integer] :timeout_interval number of seconds a task can
+    #     run before it is considered to have failed (default: TIMEOUT_INTERVAL)
+    #   @option opts [Boolean] :run_now Whether to run the task immediately
+    #     upon instantiation or to wait until the first #  execution_interval
+    #     has passed (default: false)
+    #
+    #   @!macro deref_options
+    #
+    #   @raise ArgumentError when no block is given.
+    #
+    #   @yield to the block after :execution_interval seconds have passed since
+    #     the last yield
+    #   @yieldparam task a reference to the `TimerTask` instance so that the
+    #     block can control its own lifecycle. Necessary since `self` will
+    #     refer to the execution context of the block rather than the running
+    #     `TimerTask`.
+    #
+    #   @return [TimerTask] the new `TimerTask`
+    def initialize(opts = {}, &task)
+      raise ArgumentError, "no block given" unless task
 
-        self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
-      end
+      super
+      set_deref_options opts
+    end
 
-      def schedule_next_task(interval = execution_interval)
-        exec_task = ->(completion) { execute_task(completion) }
-        Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
-        nil
-      end
+    # Is the executor running?
+    #
+    # @return [Boolean] `true` when running, `false` when shutting down or shutdown
+    def running?
+      @running.true?
+    end
 
-      # @!visibility private
-      def execute_task(completion) # rubocop:disable Metrics/MethodLength
-        return nil unless @running.true?
-
-        timeout_task = -> { timeout_task(completion) }
-
-        Concurrent::ScheduledTask.execute(
-          timeout_interval,
-          args: [completion],
-          &timeout_task
-        )
-        @thread_completed = Concurrent::Event.new
-
-        @value = @reason  = nil
-        @executor.post do
-          @value = @task.call(self)
-        rescue Exception => ex # rubocop:disable Lint/RescueException
-          @reason = ex
-        ensure
-          @thread_completed.set
+    # Execute a previously created `TimerTask`.
+    #
+    # @return [TimerTask] a reference to `self`
+    #
+    # @example Instance and execute in separate steps
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> false
+    #   task.execute
+    #   task.running? #=> true
+    #
+    # @example Instance and execute in one line
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
+    #   task.running? #=> true
+    def execute
+      synchronize do
+        if @running.false?
+          @running.make_true
+          schedule_next_task(@run_now ? 0 : @execution_interval)
         end
+      end
+      self
+    end
+
+    # Create and execute a new `TimerTask`.
+    #
+    # @!macro timer_task_initialize
+    #
+    # @example
+    #   task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> true
+    def self.execute(opts = {}, &task)
+      SidekiqUniqueJobs::TimerTask.new(opts, &task).execute
+    end
 
-        @thread_completed.wait
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval
+      synchronize { @execution_interval }
+    end
 
-        if completion.try?
-          schedule_next_task
-          time = Time.now
-          observers.notify_observers do
-            [time, value, @reason]
-          end
-        end
-        nil
-      end
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval=(value)
+      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
+
+      synchronize { @execution_interval = value }
+    end
+
+    # @!attribute [rw] timeout_interval
+    # @return [Fixnum] Number of seconds the task can run before it is
+    #   considered to have failed.
+    def timeout_interval
+      synchronize { @timeout_interval }
+    end
+
+    # @!attribute [rw] timeout_interval
+    # @return [Fixnum] Number of seconds the task can run before it is
+    #   considered to have failed.
+    def timeout_interval=(value)
+      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
+
+      synchronize { @timeout_interval = value }
+    end
+
+    private :post, :<<
+
+    private
+
+    def ns_initialize(opts, &task)
+      set_deref_options(opts)
+
+      self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
+      self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
+      @run_now  = opts[:now] || opts[:run_now]
+      @executor = Concurrent::RubySingleThreadExecutor.new
+      @running  = Concurrent::AtomicBoolean.new(false)
+      @task     = task
+      @value    = nil
+
+      self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
+    end
+
+    # @!visibility private
+    def ns_shutdown_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
+    def ns_kill_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
+    def schedule_next_task(interval = execution_interval)
+      exec_task = ->(completion) { execute_task(completion) }
+      Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
+      nil
+    end
+
+    # @!visibility private
+    def execute_task(completion) # rubocop:disable Metrics/MethodLength
+      return nil unless @running.true?
 
-      # @!visibility private
-      def timeout_task(completion)
-        return unless @running.true?
-        return unless completion.try?
+      timeout_task = -> { timeout_task(completion) }
 
-        @executor.kill
-        @executor.wait_for_termination
-        @executor = Concurrent::RubySingleThreadExecutor.new
+      Concurrent::ScheduledTask.execute(
+        timeout_interval,
+        args: [completion],
+        &timeout_task
+      )
+      @thread_completed = Concurrent::Event.new
 
+      @value = @reason  = nil
+      @executor.post do
+        @value = @task.call(self)
+      rescue Exception => ex # rubocop:disable Lint/RescueException
+        @reason = ex
+      ensure
         @thread_completed.set
+      end
+
+      @thread_completed.wait
 
+      if completion.try?
         schedule_next_task
-        observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
+        time = Time.now
+        observers.notify_observers do
+          [time, value, @reason]
+        end
       end
+      nil
+    end
+
+    # @!visibility private
+    def timeout_task(completion)
+      return unless @running.true?
+      return unless completion.try?
+
+      @executor.kill
+      @executor.wait_for_termination
+      @executor = Concurrent::RubySingleThreadExecutor.new
+
+      @thread_completed.set
+
+      schedule_next_task
+      observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
     end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.16"
+  VERSION = "7.1.19"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.16
+  version: 7.1.19
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-02 00:00:00.000000000 Z
+date: 2022-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script