sidekiq-unique-jobs 7.1.8 → 8.0.3

data/lib/sidekiq_unique_jobs/testing.rb

@@ -21,16 +21,24 @@ module Sidekiq
   #
   # @param [Hash<Symbol, Object>] tmp_config the temporary config to use
   #
-  def self.use_options(tmp_config = {})
-    old_options = default_worker_options.dup
+  def self.use_options(tmp_config = {}) # rubocop:disable Metrics/MethodLength
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = tmp_config
+    else
+      default_worker_options.clear
+      self.default_worker_options = tmp_config
+    end
 
-    default_worker_options.clear
-    self.default_worker_options = tmp_config
     yield
   ensure
-    default_worker_options.clear
-    self.default_worker_options = DEFAULT_WORKER_OPTIONS
-    self.default_worker_options = old_options
+    if respond_to?(:default_job_options)
+      default_job_options.clear
+      self.default_job_options = default_job_options
+    else
+      default_worker_options.clear
+      self.default_worker_options = DEFAULT_WORKER_OPTIONS
+    end
   end
 
   #
@@ -54,20 +62,14 @@ module Sidekiq
 
         yield
       ensure
-        self.sidekiq_options_hash = Sidekiq::DEFAULT_WORKER_OPTIONS
-        sidekiq_options(old_options)
-      end
+        self.sidekiq_options_hash =
+          if Sidekiq.respond_to?(:default_job_options)
+            Sidekiq.default_job_options
+          else
+            DEFAULT_WORKER_OPTIONS
+          end
 
-      #
-      # Clears the jobs for this worker and removes all locks
-      #
-      def clear
-        jobs.each do |job|
-          SidekiqUniqueJobs::Unlockable.unlock(job)
-        end
-
-        Sidekiq::Queues[queue].clear
-        jobs.clear
+        sidekiq_options(old_options)
       end
     end
 
@@ -88,6 +90,36 @@ module Sidekiq
         super(options)
       end
 
+      #
+      # Prepends deletion of locks to clear
+      #
+      module ClassMethods
+        #
+        # Clears the jobs for this worker and removes all locks
+        #
+        def clear
+          jobs.each do |job|
+            SidekiqUniqueJobs::Unlockable.unlock(job)
+          end
+
+          super
+        end
+      end
+    end
+
+    prepend Overrides
+
+    #
+    # Prepends methods to Sidekiq::Worker
+    #
+    module ClassMethods
+      prepend Overrides::ClassMethods
+    end
+
+    #
+    # Prepends singleton methods to Sidekiq::Worker
+    #
+    module SignletonOverrides
       #
       # Clears all jobs for this worker and removes all locks
       #
@@ -98,6 +130,6 @@ module Sidekiq
       end
     end
 
-    prepend Overrides
+    singleton_class.prepend SignletonOverrides
   end
 end

data/lib/sidekiq_unique_jobs/timer_task.rb

@@ -1,25 +1,279 @@
 # frozen_string_literal: true
 
+require "concurrent/collection/copy_on_notify_observer_set"
+require "concurrent/concern/dereferenceable"
+require "concurrent/concern/observable"
+require "concurrent/atomic/atomic_boolean"
+require "concurrent/executor/executor_service"
+require "concurrent/executor/ruby_executor_service"
+require "concurrent/executor/safe_task_executor"
+require "concurrent/scheduled_task"
+
 module SidekiqUniqueJobs
-  # @see [Concurrent::TimerTask] https://www.rubydoc.info/gems/concurrent-ruby/Concurrent/TimerTask
+  # A very common concurrency pattern is to run a thread that performs a task at
+  # regular intervals. The thread that performs the task sleeps for the given
+  # interval then wakes up and performs the task. Lather, rinse, repeat... This
+  # pattern causes two problems. First, it is difficult to test the business
+  # logic of the task because the task itself is tightly coupled with the
+  # concurrency logic. Second, an exception raised while performing the task can
+  # cause the entire thread to abend. In a long-running application where the
+  # task thread is intended to run for days/weeks/years a crashed task thread
+  # can pose a significant problem. `TimerTask` alleviates both problems.
+  #
+  # When a `TimerTask` is launched it starts a thread for monitoring the
+  # execution interval. The `TimerTask` thread does not perform the task,
+  # however. Instead, the TimerTask launches the task on a separate thread.
+  # Should the task experience an unrecoverable crash only the task thread will
+  # crash. This makes the `TimerTask` very fault tolerant. Additionally, the
+  # `TimerTask` thread can respond to the success or failure of the task,
+  # performing logging or ancillary operations.
+  #
+  # One other advantage of `TimerTask` is that it forces the business logic to
+  # be completely decoupled from the concurrency logic. The business logic can
+  # be tested separately then passed to the `TimerTask` for scheduling and
+  # running.
+  #
+  # In some cases it may be necessary for a `TimerTask` to affect its own
+  # execution cycle. To facilitate this, a reference to the TimerTask instance
+  # is passed as an argument to the provided block every time the task is
+  # executed.
+  #
+  # The `TimerTask` class includes the `Dereferenceable` mixin module so the
+  # result of the last execution is always available via the `#value` method.
+  # Dereferencing options can be passed to the `TimerTask` during construction or
+  # at any later time using the `#set_deref_options` method.
+  #
+  # `TimerTask` supports notification through the Ruby standard library
+  # {http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # Observable} module. On execution the `TimerTask` will notify the observers
+  # with three arguments: time of execution, the result of the block (or nil on
+  # failure), and any raised exceptions (or nil on success).
+  #
+  # @!macro copy_options
+  #
+  # @example Basic usage
+  #   task = Concurrent::TimerTask.new{ puts 'Boom!' }
+  #   task.execute
+  #
+  #   task.execution_interval #=> 60 (default)
+  #
+  #   # wait 60 seconds...
+  #   #=> 'Boom!'
+  #
+  #   task.shutdown #=> true
+  #
+  # @example Configuring `:execution_interval`
+  #   task = Concurrent::TimerTask.new(execution_interval: 5) do
+  #          puts 'Boom!'
+  #        end
+  #
+  #   task.execution_interval #=> 5
+  #
+  # @example Immediate execution with `:run_now`
+  #   task = Concurrent::TimerTask.new(run_now: true){ puts 'Boom!' }
+  #   task.execute
+  #
+  #   #=> 'Boom!'
+  #
+  # @example Last `#value` and `Dereferenceable` mixin
+  #   task = Concurrent::TimerTask.new(
+  #     dup_on_deref: true,
+  #     execution_interval: 5
+  #   ){ Time.now }
   #
-  class TimerTask < ::Concurrent::TimerTask
+  #   task.execute
+  #   Time.now   #=> 2013-11-07 18:06:50 -0500
+  #   sleep(10)
+  #   task.value #=> 2013-11-07 18:06:55 -0500
+  #
+  # @example Controlling execution from within the block
+  #   timer_task = Concurrent::TimerTask.new(execution_interval: 1) do |task|
+  #     task.execution_interval.times{ print 'Boom! ' }
+  #     print "\n"
+  #     task.execution_interval += 1
+  #     if task.execution_interval > 5
+  #       puts 'Stopping...'
+  #       task.shutdown
+  #     end
+  #   end
+  #
+  #   timer_task.execute # blocking call - this task will stop itself
+  #   #=> Boom!
+  #   #=> Boom! Boom!
+  #   #=> Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom!
+  #   #=> Boom! Boom! Boom! Boom! Boom!
+  #   #=> Stopping...
+  #
+  # @example Observation
+  #   class TaskObserver
+  #     def update(time, result, ex)
+  #       if result
+  #         print "(#{time}) Execution successfully returned #{result}\n"
+  #       else
+  #         print "(#{time}) Execution failed with error #{ex}\n"
+  #       end
+  #     end
+  #   end
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ 42 }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #   sleep 4
+  #
+  #   #=> (2013-10-13 19:08:58 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:08:59 -0400) Execution successfully returned 42
+  #   #=> (2013-10-13 19:09:00 -0400) Execution successfully returned 42
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ sleep }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:07:25 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:27 -0400) Execution timed out
+  #   #=> (2013-10-13 19:07:29 -0400) Execution timed out
+  #   task.shutdown
+  #
+  #   task = Concurrent::TimerTask.new(execution_interval: 1){ raise StandardError }
+  #   task.add_observer(TaskObserver.new)
+  #   task.execute
+  #
+  #   #=> (2013-10-13 19:09:37 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:38 -0400) Execution failed with error StandardError
+  #   #=> (2013-10-13 19:09:39 -0400) Execution failed with error StandardError
+  #   task.shutdown
+  #
+  # @see http://ruby-doc.org/stdlib-2.0/libdoc/observer/rdoc/Observable.html
+  # @see http://docs.oracle.com/javase/7/docs/api/java/util/TimerTask.html
+  class TimerTask < Concurrent::RubyExecutorService
+    include Concurrent::Concern::Dereferenceable
+    include Concurrent::Concern::Observable
+
+    # Default `:execution_interval` in seconds.
+    EXECUTION_INTERVAL = 60
+
+    # Default `:timeout_interval` in seconds.
+    TIMEOUT_INTERVAL = 30
+
+    # Create a new TimerTask with the given task and configuration.
+    #
+    # @!macro timer_task_initialize
+    #   @param [Hash] opts the options defining task execution.
+    #   @option opts [Integer] :execution_interval number of seconds between
+    #     task executions (default: EXECUTION_INTERVAL)
+    #   @option opts [Boolean] :run_now Whether to run the task immediately
+    #     upon instantiation or to wait until the first #  execution_interval
+    #     has passed (default: false)
+    #
+    #   @!macro deref_options
+    #
+    #   @raise ArgumentError when no block is given.
+    #
+    #   @yield to the block after :execution_interval seconds have passed since
+    #     the last yield
+    #   @yieldparam task a reference to the `TimerTask` instance so that the
+    #     block can control its own lifecycle. Necessary since `self` will
+    #     refer to the execution context of the block rather than the running
+    #     `TimerTask`.
+    #
+    #   @return [TimerTask] the new `TimerTask`
+    def initialize(opts = {}, &task)
+      raise ArgumentError, "no block given" unless task
+
+      super
+      set_deref_options opts
+    end
+
+    # Is the executor running?
+    #
+    # @return [Boolean] `true` when running, `false` when shutting down or shutdown
+    def running?
+      @running.true?
+    end
+
+    # Execute a previously created `TimerTask`.
+    #
+    # @return [TimerTask] a reference to `self`
+    #
+    # @example Instance and execute in separate steps
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> false
+    #   task.execute
+    #   task.running? #=> true
+    #
+    # @example Instance and execute in one line
+    #   task = Concurrent::TimerTask.new(execution_interval: 10){ print "Hello World\n" }.execute
+    #   task.running? #=> true
+    def execute
+      synchronize do
+        if @running.false?
+          @running.make_true
+          schedule_next_task(@run_now ? 0 : @execution_interval)
+        end
+      end
+      self
+    end
+
+    # Create and execute a new `TimerTask`.
+    #
+    # @!macro timer_task_initialize
+    #
+    # @example
+    #   task = Concurrent::TimerTask.execute(execution_interval: 10){ print "Hello World\n" }
+    #   task.running? #=> true
+    def self.execute(opts = {}, &task)
+      TimerTask.new(opts, &task).execute
+    end
+
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval
+      synchronize { @execution_interval }
+    end
+
+    # @!attribute [rw] execution_interval
+    # @return [Fixnum] Number of seconds after the task completes before the
+    #   task is performed again.
+    def execution_interval=(value)
+      raise ArgumentError, "must be greater than zero" if (value = value.to_f) <= 0.0
+
+      synchronize { @execution_interval = value }
+    end
+
+    private :post, :<<
+
     private
 
     def ns_initialize(opts, &task)
       set_deref_options(opts)
 
       self.execution_interval = opts[:execution] || opts[:execution_interval] || EXECUTION_INTERVAL
-      self.timeout_interval = opts[:timeout] || opts[:timeout_interval] || TIMEOUT_INTERVAL
-      @run_now  = opts[:now] || opts[:run_now]
-      @executor = Concurrent::RubySingleThreadExecutor.new
-      @running  = Concurrent::AtomicBoolean.new(false)
-      @task     = task
-      @value    = nil
+      if opts[:timeout] || opts[:timeout_interval]
+        warn "TimeTask timeouts are now ignored as these were not able to be implemented correctly"
+      end
+      @run_now = opts[:now] || opts[:run_now]
+      @executor = Concurrent::SafeTaskExecutor.new(task)
+      @running = Concurrent::AtomicBoolean.new(false)
+      @value = nil
 
       self.observers = Concurrent::Collection::CopyOnNotifyObserverSet.new
     end
 
+    # @!visibility private
+    def ns_shutdown_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
+    def ns_kill_execution
+      @running.make_false
+      super
+    end
+
+    # @!visibility private
     def schedule_next_task(interval = execution_interval)
       exec_task = ->(completion) { execute_task(completion) }
       Concurrent::ScheduledTask.execute(interval, args: [Concurrent::Event.new], &exec_task)
@@ -27,52 +281,19 @@ module SidekiqUniqueJobs
     end
 
     # @!visibility private
-    def execute_task(completion) # rubocop:disable Metrics/MethodLength
+    def execute_task(completion)
       return nil unless @running.true?
 
-      timeout_task = -> { timeout_task(completion) }
-
-      Concurrent::ScheduledTask.execute(
-        timeout_interval,
-        args: [completion],
-        &timeout_task
-      )
-      @thread_completed = Concurrent::Event.new
-
-      @value = @reason  = nil
-      @executor.post do
-        @value = @task.call(self)
-      rescue Exception => ex # rubocop:disable Lint/RescueException
-        @reason = ex
-      ensure
-        @thread_completed.set
-      end
-
-      @thread_completed.wait
-
+      _success, value, reason = @executor.execute(self)
       if completion.try?
+        self.value = value
         schedule_next_task
         time = Time.now
         observers.notify_observers do
-          [time, value, @reason]
+          [time, self.value, reason]
         end
       end
       nil
     end
-
-    # @!visibility private
-    def timeout_task(completion)
-      return unless @running.true?
-      return unless completion.try?
-
-      @executor.kill
-      @executor.wait_for_termination
-      @executor = Concurrent::RubySingleThreadExecutor.new
-
-      @thread_completed.set
-
-      schedule_next_task
-      observers.notify_observers(Time.now, nil, Concurrent::TimeoutError.new)
-    end
   end
 end

data/lib/sidekiq_unique_jobs/timing.rb

@@ -48,7 +48,7 @@ module SidekiqUniqueJobs
     # @return [Float]
     #
     def clock_stamp
-      if Process.const_defined?("CLOCK_MONOTONIC")
+      if Process.const_defined?(:CLOCK_MONOTONIC)
         Process.clock_gettime(Process::CLOCK_MONOTONIC)
       else
         now_f

data/lib/sidekiq_unique_jobs/upgrade_locks.rb

@@ -6,7 +6,7 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
-  class UpgradeLocks # rubocop:disable Metrics/ClassLength
+  class UpgradeLocks
     #
     # @return [Integer] the number of keys to batch upgrade
     BATCH_SIZE = 100
@@ -56,9 +56,9 @@ module SidekiqUniqueJobs
 
         log_info("Start - Upgrading Locks")
 
-        upgrade_v6_locks
-        delete_unused_v6_keys
-        delete_supporting_v6_keys
+        # upgrade_v6_locks
+        # delete_unused_v6_keys
+        # delete_supporting_v6_keys
 
         conn.hset(upgraded_key, version, now_f)
         log_info("Done - Upgrading Locks")
@@ -75,10 +75,11 @@ module SidekiqUniqueJobs
 
     def upgrade_v6_locks
       log_info("Start - Converting v6 locks to v7")
-      conn.scan_each(match: "*:GRABBED", count: BATCH_SIZE) do |grabbed_key|
+      conn.scan(match: "*:GRABBED", count: BATCH_SIZE).each do |grabbed_key|
         upgrade_v6_lock(grabbed_key)
         @count += 1
       end
+
       log_info("Done - Converting v6 locks to v7")
     end
 
@@ -87,9 +88,9 @@ module SidekiqUniqueJobs
       digest     = grabbed_key.gsub(":GRABBED", "")
       locks      = conn.hgetall(grabbed_key)
 
-      conn.pipelined do
-        conn.hmset(locked_key, *locks.to_a)
-        conn.zadd(DIGESTS, locks.values.first, digest)
+      conn.pipelined do |pipeline|
+        pipeline.hmset(locked_key, *locks.to_a)
+        pipeline.zadd(DIGESTS, locks.values.first, digest)
       end
     end
 
@@ -114,12 +115,8 @@ module SidekiqUniqueJobs
     def batch_delete(*keys)
       return if keys.empty?
 
-      conn.pipelined do
-        if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
-          conn.unlink(*keys)
-        else
-          conn.del(*keys)
-        end
+      conn.pipelined do |pipeline|
+        pipeline.unlink(*keys)
       end
     end
 

data/lib/sidekiq_unique_jobs/version.rb

@@ -3,5 +3,5 @@
 module SidekiqUniqueJobs
   #
   # @return [String] the current SidekiqUniqueJobs version
-  VERSION = "7.1.8"
+  VERSION = "8.0.3"
 end

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -27,7 +27,7 @@ module SidekiqUniqueJobs
       # @return [String] the file contents of the template
       #
       def unique_template(name)
-        File.open(unique_filename(name)).read
+        File.read(unique_filename(name))
       end
 
       #
@@ -51,6 +51,16 @@ module SidekiqUniqueJobs
         @digests ||= SidekiqUniqueJobs::Digests.new
       end
 
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::ExpiringDigests] the sorted set with expiring digests
+      #
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
       #
       # The collection of changelog entries
       #
@@ -70,11 +80,11 @@ module SidekiqUniqueJobs
       #
       def cparams(options)
         stringified_options = options.transform_keys(&:to_s)
-        params.merge(stringified_options).map do |key, value|
+        params.merge(stringified_options).filter_map do |key, value|
           next unless SAFE_CPARAMS.include?(key)
 
           "#{key}=#{CGI.escape(value.to_s)}"
-        end.compact.join("&")
+        end.join("&")
       end
 
       #
@@ -136,6 +146,8 @@ module SidekiqUniqueJobs
       # @return [String] a html safe string with relative time information
       #
       def safe_relative_time(time)
+        return unless time
+
         time = parse_time(time)
 
         relative_time(time)

data/lib/sidekiq_unique_jobs/web/views/changelogs.erb

@@ -1,54 +1,60 @@
 <header class="row">
   <div class="col-sm-5">
     <h3>
-    <%= t('Changelog Entries') %>
+      <%= t('Changelog Entries') %>
     </h3>
   </div>
   <form action="<%= root_path %>changelogs" class="form form-inline" method="get">
     <%= csrf_tag %>
+
     <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
+
     <button class="btn btn-default" type="submit">
-    <%= t('Filter') %>
+      <%= t('Filter') %>
     </button>
   </form>
+
   <% if @changelogs.any? && @total_size > @count.to_i %>
-  <div class="col-sm-4">
-    <%= erb unique_template(:_paging), locals: { url: "#{root_path}changelogs" } %>
-  </div>
+    <div class="col-sm-4">
+      <%= erb unique_template(:_paging), locals: { url: "#{root_path}changelogs" } %>
+    </div>
   <% end %>
 </header>
+
 <% if @changelogs.any? %>
-<div class="table_container">
-  <form action="<%= root_path %>changelogs/delete_all" method="get">
-    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
-  </form>
-  <br/>
-  <table class="table table-striped table-bordered table-hover">
-    <thead>
-      <tr>
-        <th><%= t('Time') %></th>
-        <th><%= t('Digest') %></th>
-        <th><%= t('Script') %></th>
-        <th><%= t('JID') %></th>
-        <th><%= t('Prev JID') %></th>
-        <th><%= t('Message') %></th>
-      </tr>
-    </thead>
-    <tbody>
-    <% @changelogs.each do |changelog| %>
-      <tr  class="changelog-row">
-        <td><%= safe_relative_time(changelog["time"]) %></td>
-        <td><%= changelog["digest"] %></td>
-        <td><%= changelog["script"] %></td>
-        <td><%= changelog["job_id"] %></td>
-        <td><%= changelog["prev_jid"] %></td>
-        <td><%= changelog["message"] %></th>
-      </tr>
-    <% end %>
-    </tbody>
-  </table>
-  <form action="<%= root_path %>changelogs/delete_all" method="get">
-    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
-  </form>
-</div>
+  <div class="table_container">
+    <form action="<%= root_path %>changelogs/delete_all" method="get">
+      <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+    </form>
+    <br/>
+
+    <table class="table table-striped table-bordered table-hover">
+      <thead>
+        <tr>
+          <th><%= t('Time') %></th>
+          <th><%= t('Digest') %></th>
+          <th><%= t('Script') %></th>
+          <th><%= t('JID') %></th>
+          <th><%= t('Prev JID') %></th>
+          <th><%= t('Message') %></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @changelogs.each do |changelog| %>
+          <tr  class="changelog-row">
+            <td><%= safe_relative_time(changelog['time']) || "bogus" %></td>
+            <td><%= changelog["digest"] %></td>
+            <td><%= changelog["script"] %></td>
+            <td><%= changelog["job_id"] %></td>
+            <td><%= changelog["prev_jid"] %></td>
+            <td><%= changelog["message"] %></th>
+          </tr>
+        <% end %>
+      </tbody>
+    </table>
+
+    <form action="<%= root_path %>changelogs/delete_all" method="get">
+      <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+    </form>
+  </div>
 <% end %>