sidekiq-unique-jobs 6.0.25 → 7.0.0.beta2

data/lib/sidekiq_unique_jobs/update_version.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Class UpdateVersion sets the right version in redis
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class UpdateVersion
+    #
+    # Sets the right versions in redis
+    #
+    # @note the version isn't used yet but will be for automatic upgrades
+    #
+    # @return [true] when version changed
+    #
+    def self.call
+      Script::Caller.call_script(
+        :update_version,
+        keys: [LIVE_VERSION, DEAD_VERSION],
+        argv: [SidekiqUniqueJobs.version],
+      )
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/upgrade_locks.rb

@@ -0,0 +1,151 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Upgrades locks between gem version upgrades
+  #
+  # @author Mikael Henriksson <mikael@zoolutions.se>
+  #
+  class UpgradeLocks # rubocop:disable Metrics/ClassLength
+    BATCH_SIZE = 100
+    OLD_SUFFIXES = %w[
+      GRABBED
+      AVAILABLE
+      EXISTS
+      VERSION
+    ].freeze
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def self.call
+      redis do |conn|
+        new(conn).call
+      end
+    end
+
+    attr_reader :conn
+
+    def initialize(conn)
+      @count         = 0
+      @conn          = conn
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def call
+      with_logging_context do
+        return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
+        # TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
+        return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
+
+        log_info("Start - Upgrading Locks")
+
+        upgrade_v6_locks
+        delete_unused_v6_keys
+        delete_supporting_v6_keys
+
+        conn.hset(upgraded_key, version, now_f)
+        log_info("Done - Upgrading Locks")
+      end
+
+      @count
+    end
+
+    private
+
+    def upgraded_key
+      @upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
+    end
+
+    def upgrade_v6_locks
+      log_info("Start - Converting v6 locks to v7")
+      conn.scan_each(match: "*:GRABBED", count: BATCH_SIZE) do |grabbed_key|
+        upgrade_v6_lock(grabbed_key)
+        @count += 1
+      end
+      log_info("Done - Converting v6 locks to v7")
+    end
+
+    def upgrade_v6_lock(grabbed_key)
+      locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
+      digest     = grabbed_key.gsub(":GRABBED", "")
+      locks      = conn.hgetall(grabbed_key)
+
+      conn.pipelined do
+        conn.hmset(locked_key, *locks.to_a)
+        conn.zadd(DIGESTS, locks.values.first, digest)
+      end
+    end
+
+    def delete_unused_v6_keys
+      log_info("Start - Deleting v6 keys")
+      OLD_SUFFIXES.each do |suffix|
+        delete_suffix(suffix)
+      end
+      log_info("Done - Deleting v6 keys")
+    end
+
+    def delete_supporting_v6_keys
+      batch_delete("unique:keys")
+    end
+
+    def delete_suffix(suffix)
+      batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
+        batch_delete(*keys)
+      end
+    end
+
+    def batch_delete(*keys)
+      return if keys.empty?
+
+      conn.pipelined do
+        if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+          conn.unlink(*keys)
+        else
+          conn.del(*keys)
+        end
+      end
+    end
+
+    def batch_scan(match:, count:)
+      cursor = "0"
+      loop do
+        cursor, values = conn.scan(cursor, match: match, per: count)
+        yield values
+        break if cursor == "0"
+      end
+    end
+
+    def version
+      SidekiqUniqueJobs.version
+    end
+
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+
+    def logging_context
+      if logger_context_hash?
+        { "uniquejobs" => :upgrade_locks }
+      else
+        "uniquejobs-upgrade_locks"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = "6.0.25"
+  #
+  # @return [String] the current SidekiqUniqueJobs version
+  VERSION = "7.0.0.beta2"
 end

data/lib/sidekiq_unique_jobs/version_check.rb

@@ -7,7 +7,7 @@ module SidekiqUniqueJobs
   # @author Mikael Henriksson <mikael@zoolutions.se>
   #
   class VersionCheck
-    PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength
+    PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Metrics/LineLength
 
     #
     # Checks if a version is consrtaint is satisfied

data/lib/sidekiq_unique_jobs/web.rb

@@ -1,10 +1,8 @@
 # frozen_string_literal: true
 
 begin
-  require "delegate"
-  require "rack"
   require "sidekiq/web"
-rescue LoadError
+rescue LoadError # rubocop:disable Lint/HandleExceptions
   # client-only usage
 end
 
@@ -16,38 +14,46 @@ module SidekiqUniqueJobs
   #
   # @author Mikael Henriksson <mikael@zoolutions.se>
   module Web
-    def self.registered(app) # rubocop:disable Metrics/MethodLength
+    def self.registered(app) # rubocop:disable Metrics/MethodLength, Metrics/AbcSize
       app.helpers do
         include Web::Helpers
       end
 
-      app.get "/unique_digests" do
+      app.get "/locks" do
         @filter         = params[:filter] || "*"
         @filter         = "*" if @filter == ""
         @count          = (params[:count] || 100).to_i
         @current_cursor = params[:cursor]
         @prev_cursor    = params[:prev_cursor]
-        @total_size, @next_cursor, @unique_digests =
-          SidekiqUniqueJobs::Digests.page(pattern: @filter, cursor: @current_cursor, page_size: @count)
+        @pagination     = { pattern: @filter, cursor: @current_cursor, page_size: @count }
+        @total_size, @next_cursor, @locks = digests.page(@pagination)
 
-        erb(unique_template(:unique_digests))
+        erb(unique_template(:locks))
       end
 
-      app.get "/unique_digests/delete_all" do
-        SidekiqUniqueJobs::Digests.delete_by_pattern("*", count: SidekiqUniqueJobs::Digests.count)
-        redirect_to :unique_digests
+      app.get "/locks/delete_all" do
+        digests.del(pattern: "*", count: digests.count)
+        redirect_to :locks
       end
 
-      app.get "/unique_digests/:digest" do
+      app.get "/locks/:digest" do
         @digest = params[:digest]
-        @unique_keys = SidekiqUniqueJobs::Util.keys("#{@digest}*", 1000)
+        @lock   = SidekiqUniqueJobs::Lock.new(@digest)
 
-        erb(unique_template(:unique_digest))
+        erb(unique_template(:lock))
       end
 
-      app.get "/unique_digests/:digest/delete" do
-        SidekiqUniqueJobs::Digests.delete_by_digest(params[:digest])
-        redirect_to :unique_digests
+      app.get "/locks/:digest/delete" do
+        digests.del(digest: params[:digest])
+        redirect_to :locks
+      end
+
+      app.get "/locks/:digest/jobs/:job_id/delete" do
+        @digest = params[:digest]
+        @lock   = SidekiqUniqueJobs::Lock.new(@digest)
+        @lock.unlock(params[:job_id])
+
+        redirect_to "locks/#{@lock.key}"
       end
     end
   end
@@ -55,6 +61,6 @@ end
 
 if defined?(Sidekiq::Web)
   Sidekiq::Web.register SidekiqUniqueJobs::Web
-  Sidekiq::Web.tabs["Unique Digests"] = "unique_digests"
-  # Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), 'locales')
+  Sidekiq::Web.tabs["Locks"] = "locks"
+  Sidekiq::Web.settings.locales << File.join(File.dirname(__FILE__), "locales")
 end

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -2,15 +2,45 @@
 
 module SidekiqUniqueJobs
   module Web
+    #
+    # Provides view helpers for the Sidekiq::Web extension
+    #
+    # @author Mikael Henriksson <mikael@zoolutions.se>
+    #
     module Helpers
-      VIEW_PATH = File.expand_path("../web/views", __dir__)
+      VIEW_PATH    = File.expand_path("../web/views", __dir__)
+      SAFE_CPARAMS = %w[cursor prev_cursor].freeze
+
+      module_function
 
+      #
+      # Opens a template file contained within this gem
+      #
+      # @param [Symbol] name the name of the template
+      #
+      # @return [String] the file contents of the template
+      #
       def unique_template(name)
         File.open(File.join(VIEW_PATH, "#{name}.erb")).read
       end
 
-      SAFE_CPARAMS = %w[cursor prev_cursor].freeze
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::Digests] the sorted set with digests
+      #
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
 
+      #
+      # Creates url safe parameters
+      #
+      # @param [Hash] options the key/value to parameterize
+      #
+      # @return [String] a url safe parameter string
+      #
       def cparams(options)
         # stringify
         options.keys.each do |key|
@@ -24,6 +54,35 @@ module SidekiqUniqueJobs
         end.compact.join("&")
       end
 
+      #
+      # Used to avoid incompatibility with older sidekiq versions
+      #
+      #
+      # @param [Array] args the unique arguments to display
+      # @param [Integer] truncate_after_chars
+      #
+      # @return [String] a string containing all non-truncated arguments
+      #
+      def display_unique_args(args, truncate_after_chars = 2000)
+        return "Invalid job payload, args is nil" if args.nil?
+        return "Invalid job payload, args must be an Array, not #{args.class.name}" unless args.is_a?(Array)
+
+        begin
+          args.map do |arg|
+            h(truncate(to_display(arg), truncate_after_chars))
+          end.join(", ")
+        rescue StandardError
+          "Illegal job arguments: #{h args.inspect}"
+        end
+      end
+
+      #
+      # Redirect to with falback
+      #
+      # @param [String] subpath the path to redirect to
+      #
+      # @return a redirect to the new subpath
+      #
       def redirect_to(subpath)
         if respond_to?(:to)
           # Sinatra-based web UI
@@ -34,14 +93,47 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Gets a relative time as html
+      #
+      # @param [Time] time an instance of Time
+      #
+      # @return [String] a html safe string with relative time information
+      #
+      def relative_time(time)
+        stamp = time.getutc.iso8601
+        %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
+      end
+
+      #
+      # Gets a relative time as html without crashing
+      #
+      # @param [Float, Integer, String, Time] time a representation of a timestamp
+      #
+      # @return [String] a html safe string with relative time information
+      #
       def safe_relative_time(time)
-        time = if time.is_a?(Numeric)
+        time = parse_time(time)
+
+        relative_time(time)
+      end
+
+      #
+      # Constructs a time from a number of different types
+      #
+      # @param [Float, Integer, String, Time] time a representation of a timestamp
+      #
+      # @return [Time]
+      #
+      def parse_time(time)
+        case time
+        when Time
+          time
+        when Integer, Float
           Time.at(time)
         else
-          Time.parse(time)
+          Time.parse(time.to_s)
         end
-
-        relative_time(time)
       end
     end
   end

data/lib/sidekiq_unique_jobs/web/views/lock.erb

@@ -0,0 +1,108 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+    <%= t('Lock information') %> <a class="btn btn-default btn-xs" href="<%= root_path %>locks"><%= t('GoBack') %></a>
+    </h3>
+  </div>
+  <div class="col-sm-7 table-responsive">
+    <% if @lock.info.none? %>
+    <h3>No Lock Information Available</h3>
+    <p>To use it turn the following setting on:
+      <code>SidekiqUniqueJobs.config.lock_info = true</code>
+    </p>
+    <% else %>
+    <table class="table table-striped table-bordered table-white table-hover">
+      <caption>Information about lock</caption>
+      <tbody>
+        <tr>
+          <th scope=row><%= t('Worker') %></td>
+          <td><%= @lock.info["worker"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Queue') %></td>
+          <td><%= @lock.info["queue"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Limit') %></td>
+          <td><%= @lock.info["limit"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('TTL') %></td>
+          <td><%= @lock.info["ttl"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Timeout') %></td>
+          <td><%= @lock.info["timeout"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Unique Args') %></td>
+          <td>
+            <code class="code-wrap">
+              <!-- We don't want to truncate any job arguments when viewing a single job's status page -->
+              <div class="args-extended"><%= display_unique_args(@lock.info["unique_args"], nil) %></div>
+            </code>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+    <% end %>
+  </div>
+</header>
+<div class="row">
+  <div class="col-sm-6 table-responsive">
+    <table class="table table-striped table-bordered table-hover">
+      <caption>Digest: <strong class="text-muted"><%= @lock.key %></strong></caption>
+      <thead>
+        <tr>
+          <th scope="col"><%= t('Locked JIDs') %></th>
+          <th scope="col"><%= t('Since') %></th>
+          <th scope="col"></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @lock.locked_jids(with_values: true).each do |job_id, time| %>
+        <tr>
+          <td><%= job_id %></td>
+          <td><%= safe_relative_time(time.to_f) %></td>
+          <td>
+            <form action="<%= root_path %>locks/<%= @lock.key %>/jobs/<%= job_id %>/delete" method="get">
+              <%= csrf_tag %>
+              <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Unlock') %>" data-confirm="<%= t('AreYouSure') %>" />
+            </form>
+          </td>
+        </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+  <div class="col-sm-6 table-responsive">
+    <% if @lock.changelog.count.positive? %>
+    <table class="table table-striped table-bordered table-hover">
+      <caption>Changelogs</caption>
+      <thead>
+        <tr>
+          <th scope="col"><%= t('At') %></th>
+          <th scope="col"><%= t('JID') %></th>
+          <th scope="col"><%= t('Message') %></th>
+          <th scope="col"><%= t('Script') %></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @lock.changelogs.each do |entry| %>
+        <tr>
+          <td scope="row"><%= safe_relative_time(entry["time"].to_f) %></td>
+          <td><%= entry["job_id"] %></td>
+          <td><%= entry["message"] %></td>
+          <td><%= entry["script"] %></td>
+        </tr>
+        <% end %>
+      </tbody>
+    </table>
+    <% end %>
+  </div>
+</div>
+<form action="<%= root_path %>locks/<%= @lock.key %>/delete" method="get">
+  <%= csrf_tag %>
+  <a class="btn btn-default btn-xs" href="<%= root_path %>locks"><%= t('GoBack') %></a>
+  <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>