sidekiq-unique-jobs 4.0.18 → 7.1.28

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -0,0 +1,299 @@
+# frozen_string_literal: true
+
+require "concurrent/collection/copy_on_notify_observer_set"
+require "concurrent/concern/dereferenceable"
+require "concurrent/concern/observable"
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/executor/executor_service"
+require "concurrent/executor/ruby_executor_service"
+require "concurrent/executor/safe_task_executor"
+require "concurrent/scheduled_task"
+
+module SidekiqUniqueJobs
+  # A very common concurrency pattern is to run a thread that performs a task at
+  # regular intervals. The thread that performs the task sleeps for the given
+  # interval then wakes up and performs the task. Lather, rinse, repeat... This
+  # pattern causes two problems. First, it is difficult to test the business
+  # logic of the task because the task itself is tightly coupled with the
+  # concurrency logic. Second, an exception raised while performing the task can
+  # cause the entire thread to abend. In a long-running application where the
+  # task thread is intended to run for days/weeks/years a crashed task thread
+  # can pose a significant problem. `TimerTask` alleviates both problems.
+  #
+  # When a `TimerTask` is launched it starts a thread for monitoring the
+  # execution interval. The `TimerTask` thread does not perform the task,
+  # however. Instead, the TimerTask launches the task on a separate thread.
+  # Should the task experience an unrecoverable crash only the task thread will
+  # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
+  # `TimerTask` thread can respond to the success or failure of the task,
+  # performing logging or ancillary operations.
+  #
+  # One other advantage of `TimerTask` is that it forces the business logic to
+  # be completely decoupled from the concurrency logic. The business logic can
+  # be tested separately then passed to the `TimerTask` for scheduling and
+  # running.
+  #
+  # In some cases it may be necessary for a `TimerTask` to affect its own
+  # execution cycle. To facilitate this, a reference to the TimerTask instance
+  # is passed as an argument to the provided block every time the task is
+  # executed.
+  #
+  # The `TimerTask` class includes the `Dereferenceable` mixin module so the
+  # result of the last execution is always available via the `#value` method.
+  # Dereferencing options can be passed to the `TimerTask` during construction or
+  # at any later time using the `#set_deref_options` method.
+  #
+  # `TimerTask` supports notification through the Ruby standard library
+  # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # Observable} module. On execution the `TimerTask` will notify the observers
+  # with three arguments: time of execution, the result of the block (or nil on
+  # failure), and any raised exceptions (or nil on success).
+  #
+  # @!macro copy_options
+  #
+  # @example Basic usage
+  #   task = Concurrent::TimerTask.new{ puts 'Boom!' }
+  #   task.execute
+  #
+  #   task.execution_interval #=> 60 (default)
+  #
+  #   # wait 60 seconds...
+  #   #=> 'Boom!'
+  #
+  #   task.shutdown #=> true
+  #
+  # @example Configuring `:execution_interval`
+  #   task = Concurrent::TimerTask.new(execution_interval: 5) do
+  #          puts 'Boom!'
+  #        end
+  #
+  #   task.execution_interval #=> 5
+  #
+  # @example Immediate execution with `:run_now`
+  #   task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
+  #   task.execute
+  #
+  #   #=> 'Boom!'
+  #
+  # @example Last `#value` and `Dereferenceable` mixin
+  #   task = Concurrent::TimerTask.new(
+  #     dup_on_deref: true,
+  #     execution_interval: 5
+  #   ){ Time.now }
+  #
+  #   task.execute
+  #   Time.now   #=> 2013-11-07 18:06:50 -0500
+  #   sleep(10)
+  #   task.value #=> 2013-11-07 18:06:55 -0500
+  #
+  # @example Controlling execution from within the block
+  #   timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
+  #     task.execution_interval.times{ print 'Boom! ' }
+  #     print "\n"
+  #     task.execution_interval += 1
+  #     if task.execution_interval > 5
+  #       puts 'Stopping...'
+  #       task.shutdown
+  #     end
+  #   end
+  #
+  #   timer_task.execute # blocking call - this task will stop itself
+  #   #=> Boom!
+  #   #=> Boom! Boom!
+  #   #=> Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom! Boom!
+  #   #=> Stopping...
+  #
+  # @example Observation
+  #   class TaskObserver
+  #     def update(time, result, ex)
+  #       if result
+  #         print "(#{time}) Execution successfully returned #{result}\n"
+  #       else
+  #         print "(#{time}) Execution failed with error #{ex}\n"
+  #       end
+  #     end
+  #   end
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #   sleep 4
+  #
+  #   #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:07:25 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:27 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:29 -0400) Execution timed out
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
+  #   task.shutdown
+  #
+  # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
+  class TimerTask < Concurrent::RubyExecutorService
+    include Concurrent::Concern::Dereferenceable
+    include Concurrent::Concern::Observable
+
+    # Default `:execution_interval` in seconds.
+    EXECUTION_INTERVAL = 60
+
+    # Default `:timeout_interval` in seconds.
+    TIMEOUT_INTERVAL = 30
+
+    # Create a new TimerTask with the given task and configuration.
+    #
+    # @!macro timer_task_initialize
+    #   @param [Hash] opts the options defining task execution.
+    #   @option opts [Integer] :execution_interval number of seconds between
+    #     task executions (default: EXECUTION_INTERVAL)
+    #   @option opts [Boolean] :run_now Whether to run the task immediately
+    #     upon instantiation or to wait until the first #  execution_interval
+    #     has passed (default: false)
+    #
+    #   @!macro deref_options
+    #
+    #   @raise ArgumentError when no block is given.
+    #
+    #   @yield to the block after :execution_interval seconds have passed since
+    #     the last yield
+    #   @yieldparam task a reference to the `TimerTask` instance so that the
+    #     block can control its own lifecycle. Necessary since `self` will
+    #     refer to the execution context of the block rather than the running
+    #     `TimerTask`.
+    #
+    #   @return [TimerTask] the new `TimerTask`
+    def initialize(opts = {}, &task)
+      raise ArgumentError, "no block given" unless task
+
+      super
+      set_deref_options opts
+    end
+
+    # Is the executor running?
+    #
+    # @return [Boolean] `true` when running, `false` when shutting down or shutdown
+    def running?
+      @running.true?
+    end
+
+    # Execute a previously created `TimerTask`.
+    #
+    # @return [TimerTask] a reference to `self`
+    #
+    # @example Instance and execute in separate steps
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> false
+    #   task.execute
+    #   task.running? #=> true
+    #
+    # @example Instance and execute in one line
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
+    #   task.running? #=> true
+    def execute
+      synchronize do
+        if @running.false?
+          @running.make_true
+          schedule_next_task(@run_now ? 0 : @execution_interval)
+        end
+      end
+      self
+    end
+
+    # Create and execute a new `TimerTask`.
+    #
+    # @!macro timer_task_initialize
+    #
+    # @example
+    #   task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> true
+    def self.execute(opts = {}, &task)
+      TimerTask.new(opts, &task).execute
+    end
+
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval
+      synchronize { @execution_interval }
+    end
+
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval=(value)
+      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
+
+      synchronize { @execution_interval = value }
+    end
+
+    private :post, :<<
+
+    private
+
+    def ns_initialize(opts, &task)
+      set_deref_options(opts)
+
+      self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
+      if opts[:timeout] || opts[:timeout_interval]
+        warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
+      end
+      @run_now = opts[:now] || opts[:run_now]
+      @executor = Concurrent::SafeTaskExecutor.new(task)
+      @running = Concurrent::AtomicBoolean.new(false)
+      @value = nil
+
+      self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
+    end
+
+    # @!visibility private
+    def ns_shutdown_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
+    def ns_kill_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
+    def schedule_next_task(interval = execution_interval)
+      exec_task = ->(completion) { execute_task(completion) }
+      Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
+      nil
+    end
+
+    # @!visibility private
+    def execute_task(completion)
+      return nil unless @running.true?
+
+      _success, value, reason = @executor.execute(self)
+      if completion.try?
+        self.value = value
+        schedule_next_task
+        time = Time.now
+        observers.notify_observers do
+          [time, self.value, reason]
+        end
+      end
+      nil
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/timing.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  # Handles timing of things
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  module Timing
+    module_function
+
+    #
+    # Used for timing method calls
+    #
+    #
+    # @return [yield return, Float]
+    #
+    def timed
+      start_time = time_source.call
+
+      [yield, time_source.call - start_time]
+    end
+
+    #
+    # Used to get a current representation of time as Integer
+    #
+    #
+    # @return [Integer]
+    #
+    def time_source
+      -> { (clock_stamp * 1000).to_i }
+    end
+
+    #
+    # Returns the current time as float
+    #
+    # @see SidekiqUniqueJobs.now_f
+    #
+    # @return [Float]
+    #
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    #
+    # Returns a float representation of the current time.
+    #   Either from Process or Time
+    #
+    #
+    # @return [Float]
+    #
+    def clock_stamp
+      if Process.const_defined?(:CLOCK_MONOTONIC)
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      else
+        now_f
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/unlockable.rb

@@ -1,47 +1,43 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
+  # Utility module to help manage unique keys in redis.
+  # Useful for deleting keys that for whatever reason wasn't deleted
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   module Unlockable
     module_function
 
+    # Unlocks a job.
+    # @param [Hash] item a Sidekiq job hash
     def unlock(item)
-      unlock_by_key(item[UNIQUE_DIGEST_KEY], item[JID_KEY])
-    end
-
-    def unlock_by_key(unique_key, jid, redis_pool = nil)
-      Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
-    end
-
-    def unlock_by_jid(jid, redis_pool = nil)
-      Scripts.call(:release_lock_by_jid, redis_pool, keys: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).unlock
     end
 
-    def unlock_by_arguments(_worker_class, _unique_arguments = {})
-      Scripts.call(:release_lock, redis_pool, keys: [unique_key], argv: [jid]) do |result|
-        after_unlock(result, __method__)
-      end
+    # Unlocks a job.
+    # @param [Hash] item a Sidekiq job hash
+    def unlock!(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).unlock!
     end
 
-    def after_unlock(result, calling_method)
-      case result
-      when 1
-        logger.debug { "successfully unlocked #{unique_key}" }
-        true
-      when 0
-        logger.debug { "expiring lock #{unique_key} is not owned by #{jid}" }
-        false
-      when -1
-        logger.debug { "#{unique_key} is not a known key" }
-        false
-      else
-        raise "#{calling_method} returned an unexpected value (#{result})"
-      end
+    # Deletes a lock unless it has ttl
+    #
+    # This is good for situations when a job is locked by another item
+    # @param [Hash] item a Sidekiq job hash
+    def delete(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).delete
     end
 
-    def logger
-      Sidekiq.logger
+    # Deletes a lock regardless of if it was locked or has ttl.
+    #
+    # This is good for situations when a job is locked by another item
+    # @param [Hash] item a Sidekiq job hash
+    def delete!(item)
+      SidekiqUniqueJobs::Job.add_digest(item)
+      SidekiqUniqueJobs::Locksmith.new(item).delete!
     end
   end
 end

data/lib/sidekiq_unique_jobs/update_version.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class UpdateVersion sets the right version in redis
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UpdateVersion
+    #
+    # Sets the right versions in redis
+    #
+    # @note the version isn't used yet but will be for automatic upgrades
+    #
+    # @return [true] when version changed
+    #
+    def self.call
+      Script::Caller.call_script(
+        :update_version,
+        keys: [LIVE_VERSION, DEAD_VERSION],
+        argv: [SidekiqUniqueJobs.version],
+      )
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/upgrade_locks.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Upgrades locks between gem version upgrades
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UpgradeLocks # rubocop:disable Metrics/ClassLength
+    #
+    # @return [Integer] the number of keys to batch upgrade
+    BATCH_SIZE = 100
+    #
+    # @return [Array<String>] suffixes for old version
+    OLD_SUFFIXES = %w[
+      GRABBED
+      AVAILABLE
+      EXISTS
+      VERSION
+    ].freeze
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def self.call
+      redis do |conn|
+        new(conn).call
+      end
+    end
+
+    attr_reader :conn
+
+    def initialize(conn)
+      @count         = 0
+      @conn          = conn
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def call
+      with_logging_context do
+        return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
+        # TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
+        return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
+
+        log_info("Start - Upgrading Locks")
+
+        upgrade_v6_locks
+        delete_unused_v6_keys
+        delete_supporting_v6_keys
+
+        conn.hset(upgraded_key, version, now_f)
+        log_info("Done - Upgrading Locks")
+      end
+
+      @count
+    end
+
+    private
+
+    def upgraded_key
+      @upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
+    end
+
+    def upgrade_v6_locks
+      log_info("Start - Converting v6 locks to v7")
+      conn.scan_each(match: "*:GRABBED", count: BATCH_SIZE) do |grabbed_key|
+        upgrade_v6_lock(grabbed_key)
+        @count += 1
+      end
+      log_info("Done - Converting v6 locks to v7")
+    end
+
+    def upgrade_v6_lock(grabbed_key)
+      locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
+      digest     = grabbed_key.gsub(":GRABBED", "")
+      locks      = conn.hgetall(grabbed_key)
+
+      conn.pipelined do |pipeline|
+        pipeline.hmset(locked_key, *locks.to_a)
+        pipeline.zadd(DIGESTS, locks.values.first, digest)
+      end
+    end
+
+    def delete_unused_v6_keys
+      log_info("Start - Deleting v6 keys")
+      OLD_SUFFIXES.each do |suffix|
+        delete_suffix(suffix)
+      end
+      log_info("Done - Deleting v6 keys")
+    end
+
+    def delete_supporting_v6_keys
+      batch_delete("unique:keys")
+    end
+
+    def delete_suffix(suffix)
+      batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
+        batch_delete(*keys)
+      end
+    end
+
+    def batch_delete(*keys)
+      return if keys.empty?
+
+      conn.pipelined do |pipeline|
+        if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+          pipeline.unlink(*keys)
+        else
+          pipeline.del(*keys)
+        end
+      end
+    end
+
+    def batch_scan(match:, count:)
+      cursor = "0"
+      loop do
+        cursor, values = conn.scan(cursor, match: match, count: count)
+        yield values
+        break if cursor == "0"
+      end
+    end
+
+    def version
+      SidekiqUniqueJobs.version
+    end
+
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+
+    def logging_context
+      if logger_context_hash?
+        { "uniquejobs" => :upgrade_locks }
+      else
+        "uniquejobs-upgrade_locks"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
 module SidekiqUniqueJobs
-  VERSION = '4.0.18'.freeze
+  #
+  # @return [String] the current SidekiqUniqueJobs version
+  VERSION = "7.1.28"
 end