sidekiq-unique-jobs 7.1.17 → 7.1.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 787e3a8e85324b12df2f67dc04bee3ad3ad26deadc06ce170c9a5c5be62c1047
-  data.tar.gz: d280fac7bf32f98bbe8071822463c7184e24339ad756953131d113f2b18710a3
+  metadata.gz: 2a112ea37ca98a7909abf3b536e5d5f2b147e3a40f7749bf3c7d819c259d069e
+  data.tar.gz: a3f34c72a78723c0dd441d005ae0300662183ddbe24cde430e7714587feccbd3
 SHA512:
-  metadata.gz: c6549578ab2200965dcbab5e8acbf9e159f5dccce5bd2eb277b6914739e3279195169fd59349d47ae8c2cf9dec2bb7b2116fbceafd17d442b1be6096f9e559fb
-  data.tar.gz: e2469b5c19d39d97fb231db49f291a8e5dcd6763f39c28d794b69b43f1551ffc5275075b39a6d11a5eda82409f87f3d4a0d5d0ce2789b4cf90f84c5940c58c2c
+  metadata.gz: 0a6d73f2beade75f9b0261a65ff3be37dfe0fe35c6ff3550635d3b8c634f1ad80c646c36be63c304806ef4f4f471eb0bb56ad641714ed48a53a14ee1d9f1a1aa
+  data.tar.gz: 93f518d9743445c8b844032dbcd8e583a88909b7e3bed9b88807ce3e3e337d2bdf101b8b413e8610705a22aae98747c8fa63549a2b001adc9800903577ad3694

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Changelog
 
+## [v7.1.19](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.19) (2022-04-09)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.18...v7.1.19)
+
+**Fixed bugs:**
+
+- concurrent-ruby 1.1.10 spikes volume of jobs [\#701](https://github.com/mhenrixon/sidekiq-unique-jobs/issues/701)
+- Reimplement the entire TimerTask as it was [\#702](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/702) ([mhenrixon](https://github.com/mhenrixon))
+
+## [v7.1.18](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.18) (2022-04-05)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.17...v7.1.18)
+
+**Implemented enhancements:**
+
+- Make sure we reflect on execution failure [\#700](https://github.com/mhenrixon/sidekiq-unique-jobs/pull/700) ([mhenrixon](https://github.com/mhenrixon))
+
+## [v7.1.17](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.17) (2022-04-05)
+
+[Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.16...v7.1.17)
+
 ## [v7.1.16](https://github.com/mhenrixon/sidekiq-unique-jobs/tree/v7.1.16) (2022-04-02)
 
 [Full Changelog](https://github.com/mhenrixon/sidekiq-unique-jobs/compare/v7.1.15...v7.1.16)

data/README.md

@@ -123,9 +123,11 @@ Before v7, the middleware was configured automatically. Since some people report
 
 *NOTE* if you want to use the reaper you also need to configure the server middleware.
 
-[A full example](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/myapp/config/initializers/sidekiq.rb#L12)
+The following shows how to modify your `config/initializers/sidekiq.rb` file to use the middleware. [Here is a full example.](https://github.com/mhenrixon/sidekiq-unique-jobs/blob/master/myapp/config/initializers/sidekiq.rb#L12)
 
 ```ruby
+require "sidekiq-unique-jobs"
+
 Sidekiq.configure_server do |config|
   config.redis = { url: ENV["REDIS_URL"], driver: :hiredis }
 

data/lib/sidekiq_unique_jobs/orphans/manager.rb

@@ -110,13 +110,7 @@ module SidekiqUniqueJobs
       # @return [Hash]
       #
       def timer_task_options
-        timer_task_options = { run_now: true, execution_interval: reaper_interval }
-
-        if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
-          timer_task_options[:timeout_interval] = reaper_timeout
-        end
-
-        timer_task_options
+        { run_now: true, execution_interval: reaper_interval }
       end
 
       #
@@ -133,13 +127,6 @@ module SidekiqUniqueJobs
         SidekiqUniqueJobs.config.reaper_interval
       end
 
-      #
-      # @see SidekiqUniqueJobs::Config#reaper_timeout
-      #
-      def reaper_timeout
-        SidekiqUniqueJobs.config.reaper_timeout
-      end
-
       #
       # A context to use for all log entries
       #

data/lib/sidekiq_unique_jobs/orphans/ruby_reaper.rb

@@ -11,6 +11,8 @@ module SidekiqUniqueJobs
     #
     # rubocop:disable Metrics/ClassLength
     class RubyReaper < Reaper
+      include SidekiqUniqueJobs::Timing
+
       #
       # @return [String] the suffix for :RUN locks
       RUN_SUFFIX = ":RUN"
@@ -30,6 +32,16 @@ module SidekiqUniqueJobs
       #   @return [Redis::SortedSet] the Sidekiq RetrySet
       attr_reader :retried
 
+      #
+      # @!attribute [r] start_time
+      #   @return [Integer] The timestamp this execution started represented as integer
+      attr_reader :start_time
+
+      #
+      # @!attribute [r] timeout_ms
+      #   @return [Integer] The allowed ms before timeout
+      attr_reader :timeout_ms
+
       #
       # Initialize a new instance of DeleteOrphans
       #
@@ -37,9 +49,11 @@ module SidekiqUniqueJobs
       #
       def initialize(conn)
         super(conn)
-        @digests   = SidekiqUniqueJobs::Digests.new
-        @scheduled = Redis::SortedSet.new(SCHEDULE)
-        @retried   = Redis::SortedSet.new(RETRY)
+        @digests    = SidekiqUniqueJobs::Digests.new
+        @scheduled  = Redis::SortedSet.new(SCHEDULE)
+        @retried    = Redis::SortedSet.new(RETRY)
+        @start_time = time_source.call
+        @timeout_ms = SidekiqUniqueJobs.config.reaper_timeout * 1000
       end
 
       #
@@ -60,7 +74,7 @@ module SidekiqUniqueJobs
       #
       # @return [Array<String>] an array of orphaned digests
       #
-      def orphans # rubocop:disable Metrics/MethodLength
+      def orphans # rubocop:disable Metrics/MethodLength, Metrics/CyclomaticComplexity
         page = 0
         per = reaper_count * 2
         orphans = []
@@ -68,12 +82,14 @@ module SidekiqUniqueJobs
 
         while results.size.positive?
           results.each do |digest|
+            break if timeout?
             next if belongs_to_job?(digest)
 
             orphans << digest
             break if orphans.size >= reaper_count
           end
 
+          break if timeout?
           break if orphans.size >= reaper_count
 
           page += 1
@@ -83,6 +99,14 @@ module SidekiqUniqueJobs
         orphans
       end
 
+      def timeout?
+        elapsed_ms >= timeout_ms
+      end
+
+      def elapsed_ms
+        time_source.call - start_time
+      end
+
       #
       # Checks if the digest has a matching job.
       #   1. It checks the scheduled set

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -1,84 +1,299 @@
 # frozen_string_literal: true
 
-require "concurrent/version"
-require_relative "version_check"
+require "concurrent/collection/copy_on_notify_observer_set"
+require "concurrent/concern/dereferenceable"
+require "concurrent/concern/observable"
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/executor/executor_service"
+require "concurrent/executor/ruby_executor_service"
+require "concurrent/executor/safe_task_executor"
+require "concurrent/scheduled_task"
 
 module SidekiqUniqueJobs
-  # @see [Concurrent::TimerTask] https://www.rubydoc.info/gems/concurrent-ruby/Concurrent/TimerTask
+  # A very common concurrency pattern is to run a thread that performs a task at
+  # regular intervals. The thread that performs the task sleeps for the given
+  # interval then wakes up and performs the task. Lather, rinse, repeat... This
+  # pattern causes two problems. First, it is difficult to test the business
+  # logic of the task because the task itself is tightly coupled with the
+  # concurrency logic. Second, an exception raised while performing the task can
+  # cause the entire thread to abend. In a long-running application where the
+  # task thread is intended to run for days/weeks/years a crashed task thread
+  # can pose a significant problem. `TimerTask` alleviates both problems.
   #
-  class TimerTask < ::Concurrent::TimerTask
-    if VersionCheck.satisfied?(::Concurrent::VERSION, "< 1.1.10")
+  # When a `TimerTask` is launched it starts a thread for monitoring the
+  # execution interval. The `TimerTask` thread does not perform the task,
+  # however. Instead, the TimerTask launches the task on a separate thread.
+  # Should the task experience an unrecoverable crash only the task thread will
+  # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
+  # `TimerTask` thread can respond to the success or failure of the task,
+  # performing logging or ancillary operations.
+  #
+  # One other advantage of `TimerTask` is that it forces the business logic to
+  # be completely decoupled from the concurrency logic. The business logic can
+  # be tested separately then passed to the `TimerTask` for scheduling and
+  # running.
+  #
+  # In some cases it may be necessary for a `TimerTask` to affect its own
+  # execution cycle. To facilitate this, a reference to the TimerTask instance
+  # is passed as an argument to the provided block every time the task is
+  # executed.
+  #
+  # The `TimerTask` class includes the `Dereferenceable` mixin module so the
+  # result of the last execution is always available via the `#value` method.
+  # Dereferencing options can be passed to the `TimerTask` during construction or
+  # at any later time using the `#set_deref_options` method.
+  #
+  # `TimerTask` supports notification through the Ruby standard library
+  # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # Observable} module. On execution the `TimerTask` will notify the observers
+  # with three arguments: time of execution, the result of the block (or nil on
+  # failure), and any raised exceptions (or nil on success).
+  #
+  # @!macro copy_options
+  #
+  # @example Basic usage
+  #   task = Concurrent::TimerTask.new{ puts 'Boom!' }
+  #   task.execute
+  #
+  #   task.execution_interval #=> 60 (default)
+  #
+  #   # wait 60 seconds...
+  #   #=> 'Boom!'
+  #
+  #   task.shutdown #=> true
+  #
+  # @example Configuring `:execution_interval`
+  #   task = Concurrent::TimerTask.new(execution_interval: 5) do
+  #          puts 'Boom!'
+  #        end
+  #
+  #   task.execution_interval #=> 5
+  #
+  # @example Immediate execution with `:run_now`
+  #   task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
+  #   task.execute
+  #
+  #   #=> 'Boom!'
+  #
+  # @example Last `#value` and `Dereferenceable` mixin
+  #   task = Concurrent::TimerTask.new(
+  #     dup_on_deref: true,
+  #     execution_interval: 5
+  #   ){ Time.now }
+  #
+  #   task.execute
+  #   Time.now   #=> 2013-11-07 18:06:50 -0500
+  #   sleep(10)
+  #   task.value #=> 2013-11-07 18:06:55 -0500
+  #
+  # @example Controlling execution from within the block
+  #   timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
+  #     task.execution_interval.times{ print 'Boom! ' }
+  #     print "\n"
+  #     task.execution_interval += 1
+  #     if task.execution_interval > 5
+  #       puts 'Stopping...'
+  #       task.shutdown
+  #     end
+  #   end
+  #
+  #   timer_task.execute # blocking call - this task will stop itself
+  #   #=> Boom!
+  #   #=> Boom! Boom!
+  #   #=> Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom! Boom!
+  #   #=> Stopping...
+  #
+  # @example Observation
+  #   class TaskObserver
+  #     def update(time, result, ex)
+  #       if result
+  #         print "(#{time}) Execution successfully returned #{result}\n"
+  #       else
+  #         print "(#{time}) Execution failed with error #{ex}\n"
+  #       end
+  #     end
+  #   end
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #   sleep 4
+  #
+  #   #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:07:25 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:27 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:29 -0400) Execution timed out
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
+  #   task.shutdown
+  #
+  # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
+  class TimerTask < Concurrent::RubyExecutorService
+    include Concurrent::Concern::Dereferenceable
+    include Concurrent::Concern::Observable
 
-      private
+    # Default `:execution_interval` in seconds.
+    EXECUTION_INTERVAL = 60
 
-      def ns_initialize(opts, &task)
-        set_deref_options(opts)
+    # Default `:timeout_interval` in seconds.
+    TIMEOUT_INTERVAL = 30
 
-        self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
-        self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
-        @run_now  = opts[:now] || opts[:run_now]
-        @executor = Concurrent::RubySingleThreadExecutor.new
-        @running  = Concurrent::AtomicBoolean.new(false)
-        @task     = task
-        @value    = nil
+    # Create a new TimerTask with the given task and configuration.
+    #
+    # @!macro timer_task_initialize
+    #   @param [Hash] opts the options defining task execution.
+    #   @option opts [Integer] :execution_interval number of seconds between
+    #     task executions (default: EXECUTION_INTERVAL)
+    #   @option opts [Boolean] :run_now Whether to run the task immediately
+    #     upon instantiation or to wait until the first #  execution_interval
+    #     has passed (default: false)
+    #
+    #   @!macro deref_options
+    #
+    #   @raise ArgumentError when no block is given.
+    #
+    #   @yield to the block after :execution_interval seconds have passed since
+    #     the last yield
+    #   @yieldparam task a reference to the `TimerTask` instance so that the
+    #     block can control its own lifecycle. Necessary since `self` will
+    #     refer to the execution context of the block rather than the running
+    #     `TimerTask`.
+    #
+    #   @return [TimerTask] the new `TimerTask`
+    def initialize(opts = {}, &task)
+      raise ArgumentError, "no block given" unless task
 
-        self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
-      end
+      super
+      set_deref_options opts
+    end
 
-      def schedule_next_task(interval = execution_interval)
-        exec_task = ->(completion) { execute_task(completion) }
-        Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
-        nil
-      end
+    # Is the executor running?
+    #
+    # @return [Boolean] `true` when running, `false` when shutting down or shutdown
+    def running?
+      @running.true?
+    end
 
-      # @!visibility private
-      def execute_task(completion) # rubocop:disable Metrics/MethodLength
-        return nil unless @running.true?
-
-        timeout_task = -> { timeout_task(completion) }
-
-        Concurrent::ScheduledTask.execute(
-          timeout_interval,
-          args: [completion],
-          &timeout_task
-        )
-        @thread_completed = Concurrent::Event.new
-
-        @value = @reason  = nil
-        @executor.post do
-          @value = @task.call(self)
-        rescue Exception => ex # rubocop:disable Lint/RescueException
-          @reason = ex
-        ensure
-          @thread_completed.set
+    # Execute a previously created `TimerTask`.
+    #
+    # @return [TimerTask] a reference to `self`
+    #
+    # @example Instance and execute in separate steps
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> false
+    #   task.execute
+    #   task.running? #=> true
+    #
+    # @example Instance and execute in one line
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
+    #   task.running? #=> true
+    def execute
+      synchronize do
+        if @running.false?
+          @running.make_true
+          schedule_next_task(@run_now ? 0 : @execution_interval)
         end
+      end
+      self
+    end
 
-        @thread_completed.wait
+    # Create and execute a new `TimerTask`.
+    #
+    # @!macro timer_task_initialize
+    #
+    # @example
+    #   task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> true
+    def self.execute(opts = {}, &task)
+      TimerTask.new(opts, &task).execute
+    end
 
-        if completion.try?
-          schedule_next_task
-          time = Time.now
-          observers.notify_observers do
-            [time, value, @reason]
-          end
-        end
-        nil
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval
+      synchronize { @execution_interval }
+    end
+
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval=(value)
+      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
+
+      synchronize { @execution_interval = value }
+    end
+
+    private :post, :<<
+
+    private
+
+    def ns_initialize(opts, &task)
+      set_deref_options(opts)
+
+      self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
+      if opts[:timeout] || opts[:timeout_interval]
+        warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
       end
+      @run_now = opts[:now] || opts[:run_now]
+      @executor = Concurrent::SafeTaskExecutor.new(task)
+      @running = Concurrent::AtomicBoolean.new(false)
+      @value = nil
+
+      self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
+    end
+
+    # @!visibility private
+    def ns_shutdown_execution
+      @running.make_false
+      super
+    end
 
-      # @!visibility private
-      def timeout_task(completion)
-        return unless @running.true?
-        return unless completion.try?
+    # @!visibility private
+    def ns_kill_execution
+      @running.make_false
+      super
+    end
 
-        @executor.kill
-        @executor.wait_for_termination
-        @executor = Concurrent::RubySingleThreadExecutor.new
+    # @!visibility private
+    def schedule_next_task(interval = execution_interval)
+      exec_task = ->(completion) { execute_task(completion) }
+      Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
+      nil
+    end
 
-        @thread_completed.set
+    # @!visibility private
+    def execute_task(completion)
+      return nil unless @running.true?
 
+      _success, value, reason = @executor.execute(self)
+      if completion.try?
+        self.value = value
         schedule_next_task
-        observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
+        time = Time.now
+        observers.notify_observers do
+          [time, self.value, reason]
+        end
       end
+      nil
     end
   end
 end

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.17"
+  VERSION = "7.1.20"
 end

data/lib/tasks/changelog.rake

@@ -17,7 +17,7 @@ COMMIT_CHANGELOG_CMD = "git commit -a -m 'Update changelog'"
 desc "Generate a Changelog"
 task :changelog do
   sh("git checkout main")
-  sh(*CHANGELOG_CMD.push(ENV["CHANGELOG_GITHUB_TOKEN"]))
+  sh(*CHANGELOG_CMD.push(ENV.fetch("CHANGELOG_GITHUB_TOKEN", nil)))
   sh(ADD_CHANGELOG_CMD)
   sh(COMMIT_CHANGELOG_CMD)
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: sidekiq-unique-jobs
 version: !ruby/object:Gem::Version
-  version: 7.1.17
+  version: 7.1.20
 platform: ruby
 authors:
 - Mikael Henriksson
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-04-05 00:00:00.000000000 Z
+date: 2022-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: brpoplpush-redis_script