sidekiq-unique-jobs 6.0.25 → 7.1.29

data/lib/sidekiq_unique_jobs/upgrade_locks.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+module SidekiqUniqueJobs
+  #
+  # Upgrades locks between gem version upgrades
+  #
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
+  #
+  class UpgradeLocks # rubocop:disable Metrics/ClassLength
+    #
+    # @return [Integer] the number of keys to batch upgrade
+    BATCH_SIZE = 100
+    #
+    # @return [Array<String>] suffixes for old version
+    OLD_SUFFIXES = %w[
+      GRABBED
+      AVAILABLE
+      EXISTS
+      VERSION
+    ].freeze
+
+    include SidekiqUniqueJobs::Logging
+    include SidekiqUniqueJobs::Connection
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def self.call
+      redis do |conn|
+        new(conn).call
+      end
+    end
+
+    attr_reader :conn
+
+    def initialize(conn)
+      @count         = 0
+      @conn          = conn
+      redis_version # Avoid pipelined calling redis_version and getting a future.
+    end
+
+    #
+    # Performs upgrade of old locks
+    #
+    #
+    # @return [Integer] the number of upgrades locks
+    #
+    def call
+      with_logging_context do
+        return log_info("Already upgraded to #{version}") if conn.hget(upgraded_key, version)
+        # TODO: Needs handling of v7.0.0 => v7.0.1 where we don't want to
+        return log_info("Skipping upgrade because #{DEAD_VERSION} has been set") if conn.get(DEAD_VERSION)
+
+        log_info("Start - Upgrading Locks")
+
+        upgrade_v6_locks
+        delete_unused_v6_keys
+        delete_supporting_v6_keys
+
+        conn.hset(upgraded_key, version, now_f)
+        log_info("Done - Upgrading Locks")
+      end
+
+      @count
+    end
+
+    private
+
+    def upgraded_key
+      @upgraded_key ||= "#{LIVE_VERSION}:UPGRADED"
+    end
+
+    def upgrade_v6_locks
+      log_info("Start - Converting v6 locks to v7")
+      conn.scan_each(match: "*:GRABBED", count: BATCH_SIZE) do |grabbed_key|
+        upgrade_v6_lock(grabbed_key)
+        @count += 1
+      end
+      log_info("Done - Converting v6 locks to v7")
+    end
+
+    def upgrade_v6_lock(grabbed_key)
+      locked_key = grabbed_key.gsub(":GRABBED", ":LOCKED")
+      digest     = grabbed_key.gsub(":GRABBED", "")
+      locks      = conn.hgetall(grabbed_key)
+
+      conn.pipelined do |pipeline|
+        pipeline.hmset(locked_key, *locks.to_a)
+        pipeline.zadd(DIGESTS, locks.values.first, digest)
+      end
+    end
+
+    def delete_unused_v6_keys
+      log_info("Start - Deleting v6 keys")
+      OLD_SUFFIXES.each do |suffix|
+        delete_suffix(suffix)
+      end
+      log_info("Done - Deleting v6 keys")
+    end
+
+    def delete_supporting_v6_keys
+      batch_delete("unique:keys")
+    end
+
+    def delete_suffix(suffix)
+      batch_scan(match: "*:#{suffix}", count: BATCH_SIZE) do |keys|
+        batch_delete(*keys)
+      end
+    end
+
+    def batch_delete(*keys)
+      return if keys.empty?
+
+      conn.pipelined do |pipeline|
+        if VersionCheck.satisfied?(redis_version, ">= 4.0.0")
+          pipeline.unlink(*keys)
+        else
+          pipeline.del(*keys)
+        end
+      end
+    end
+
+    def batch_scan(match:, count:)
+      cursor = "0"
+      loop do
+        cursor, values = conn.scan(cursor, match: match, count: count)
+        yield values
+        break if cursor == "0"
+      end
+    end
+
+    def version
+      SidekiqUniqueJobs.version
+    end
+
+    def now_f
+      SidekiqUniqueJobs.now_f
+    end
+
+    def redis_version
+      @redis_version ||= SidekiqUniqueJobs.config.redis_version
+    end
+
+    def logging_context
+      if logger_context_hash?
+        { "uniquejobs" => :upgrade_locks }
+      else
+        "uniquejobs-upgrade_locks"
+      end
+    end
+  end
+end

data/lib/sidekiq_unique_jobs/version.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
 module SidekiqUniqueJobs
-  VERSION = "6.0.25"
+  #
+  # @return [String] the current SidekiqUniqueJobs version
+  VERSION = "7.1.29"
 end

data/lib/sidekiq_unique_jobs/version_check.rb

@@ -4,13 +4,13 @@ module SidekiqUniqueJobs
   #
   # Handles checking if a version is compliant with given constraint
   #
-  # @author Mikael Henriksson <mikael@zoolutions.se>
+  # @author Mikael Henriksson <mikael@mhenrixon.com>
   #
   class VersionCheck
-    PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength
+    PATTERN = /(?<operator1>[<>=]+)?\s?(?<version1>(\d+.?)+)(\s+&&\s+)?(?<operator2>[<>=]+)?\s?(?<version2>(\d+.?)+)?/m.freeze # rubocop:disable Layout/LineLength, Lint/MixedRegexpCaptureTypes
 
     #
-    # Checks if a version is consrtaint is satisfied
+    # Checks if a version is constraint is satisfied
     #
     # @example A satisfied constraint
     #   VersionCheck.satisfied?("5.0.0", ">= 4.0.0") #=> true
@@ -22,12 +22,31 @@ module SidekiqUniqueJobs
     # @param [String] version a version string `5.0.0`
     # @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
     #
-    # @return [<type>] <description>
+    # @return [true, false] <description>
     #
     def self.satisfied?(version, constraint)
       new(version, constraint).satisfied?
     end
 
+    #
+    # Checks if a version is constraint is unfulfilled
+    #
+    # @example A satisfied constraint
+    #   VersionCheck.unfulfilled?("5.0.0", ">= 4.0.0") #=> false
+    #
+    # @example An unfulfilled constraint
+    #   VersionCheck.unfulfilled?("5.0.0", "<= 4.0.0") #=> true
+    #
+    #
+    # @param [String] version a version string `5.0.0`
+    # @param [String] constraint a version constraint `>= 5.0.0 <= 5.1.1`
+    #
+    # @return [true, false] <description>
+    #
+    def self.unfulfilled?(version, constraint)
+      !satisfied?(version, constraint)
+    end
+
     #
     # @!attribute [r] version
     #   @return [String] a version string `5.0.0`

data/lib/sidekiq_unique_jobs/web/helpers.rb

@@ -2,28 +2,120 @@
 
 module SidekiqUniqueJobs
   module Web
+    #
+    # Provides view helpers for the Sidekiq::Web extension
+    #
+    # @author Mikael Henriksson <mikael@mhenrixon.com>
+    #
     module Helpers
-      VIEW_PATH = File.expand_path("../web/views", __dir__)
+      #
+      # @return [String] the path to gem specific views
+      VIEW_PATH    = File.expand_path("../web/views", __dir__).freeze
+      #
+      # @return [Array<String>] safe params
+      SAFE_CPARAMS = %w[
+        filter count cursor prev_cursor poll direction
+      ].freeze
 
+      extend self
+
+      #
+      # Opens a template file contained within this gem
+      #
+      # @param [Symbol] name the name of the template
+      #
+      # @return [String] the file contents of the template
+      #
       def unique_template(name)
-        File.open(File.join(VIEW_PATH, "#{name}.erb")).read
+        File.read(unique_filename(name))
       end
 
-      SAFE_CPARAMS = %w[cursor prev_cursor].freeze
+      #
+      # Construct template file name
+      #
+      # @param [Symbol] name the name of the template
+      #
+      # @return [String] the full name of the file
+      #
+      def unique_filename(name)
+        File.join(VIEW_PATH, "#{name}.erb")
+      end
 
-      def cparams(options)
-        # stringify
-        options.keys.each do |key|
-          options[key.to_s] = options.delete(key)
-        end
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::Digests] the sorted set with digests
+      #
+      def digests
+        @digests ||= SidekiqUniqueJobs::Digests.new
+      end
 
-        params.merge(options).map do |key, value|
+      #
+      # The collection of digests
+      #
+      #
+      # @return [SidekiqUniqueJobs::ExpiringDigests] the sorted set with expiring digests
+      #
+      def expiring_digests
+        @expiring_digests ||= SidekiqUniqueJobs::ExpiringDigests.new
+      end
+
+      #
+      # The collection of changelog entries
+      #
+      #
+      # @return [SidekiqUniqueJobs::Digests] the sorted set with digests
+      #
+      def changelog
+        @changelog ||= SidekiqUniqueJobs::Changelog.new
+      end
+
+      #
+      # Creates url safe parameters
+      #
+      # @param [Hash] options the key/value to parameterize
+      #
+      # @return [String] a url safe parameter string
+      #
+      def cparams(options)
+        stringified_options = options.transform_keys(&:to_s)
+        params.merge(stringified_options).map do |key, value|
           next unless SAFE_CPARAMS.include?(key)
 
           "#{key}=#{CGI.escape(value.to_s)}"
         end.compact.join("&")
       end
 
+      #
+      # Used to avoid incompatibility with older sidekiq versions
+      #
+      #
+      # @param [Array] args the unique arguments to display
+      # @param [Integer] truncate_after_chars
+      #
+      # @return [String] a string containing all non-truncated arguments
+      #
+      def display_lock_args(args, truncate_after_chars = 2000)
+        return "Invalid job payload, args is nil" if args.nil?
+        return "Invalid job payload, args must be an Array, not #{args.class.name}" unless args.is_a?(Array)
+
+        begin
+          args.map do |arg|
+            h(truncate(to_display(arg), truncate_after_chars))
+          end.join(", ")
+        rescue StandardError
+          "Illegal job arguments: #{h args.inspect}"
+        end
+      end
+
+      #
+      # Redirect to with falback
+      #
+      # @param [String] subpath the path to redirect to
+      #
+      # @return a redirect to the new subpath
+      #
       def redirect_to(subpath)
         if respond_to?(:to)
           # Sinatra-based web UI
@@ -34,14 +126,47 @@ module SidekiqUniqueJobs
         end
       end
 
+      #
+      # Gets a relative time as html
+      #
+      # @param [Time] time an instance of Time
+      #
+      # @return [String] a html safe string with relative time information
+      #
+      def relative_time(time)
+        stamp = time.getutc.iso8601
+        %(<time class="ltr" dir="ltr" title="#{stamp}" datetime="#{stamp}">#{time}</time>)
+      end
+
+      #
+      # Gets a relative time as html without crashing
+      #
+      # @param [Float, Integer, String, Time] time a representation of a timestamp
+      #
+      # @return [String] a html safe string with relative time information
+      #
       def safe_relative_time(time)
-        time = if time.is_a?(Numeric)
+        time = parse_time(time)
+
+        relative_time(time)
+      end
+
+      #
+      # Constructs a time from a number of different types
+      #
+      # @param [Float, Integer, String, Time] time a representation of a timestamp
+      #
+      # @return [Time]
+      #
+      def parse_time(time)
+        case time
+        when Time
+          time
+        when Integer, Float
           Time.at(time)
         else
-          Time.parse(time)
+          Time.parse(time.to_s)
         end
-
-        relative_time(time)
       end
     end
   end

data/lib/sidekiq_unique_jobs/web/views/_paging.erb

@@ -1,10 +1,10 @@
 <ul class="pagination pull-right flip">
   <% if @prev_cursor %>
-    <li>
-      <a href="<%= url %>?<%= cparams(cursor: @prev_cursor, prev_cursor: @next_cursor) %>">Previous <%= @count %></a>
-    </li>
+  <li>
+    <a href="<%= url %>?<%= cparams(filter: @filter, cursor: @prev_cursor, prev_cursor: @next_cursor) %>">Previous <%= @count %></a>
+  </li>
   <% end %>
   <li>
-    <a href="<%= url %>?<%= cparams(cursor: @next_cursor, prev_cursor: @current_cursor) %>">Next <%= @count %></a>
+    <a href="<%= url %>?<%= cparams(filter: @filter, cursor: @next_cursor, prev_cursor: @current_cursor) %>">Next <%= @count %></a>
   </li>
 </ul>

data/lib/sidekiq_unique_jobs/web/views/changelogs.erb

@@ -0,0 +1,54 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+    <%= t('Changelog Entries') %>
+    </h3>
+  </div>
+  <form action="<%= root_path %>changelogs" class="form form-inline" method="get">
+    <%= csrf_tag %>
+    <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
+    <button class="btn btn-default" type="submit">
+    <%= t('Filter') %>
+    </button>
+  </form>
+  <% if @changelogs.any? && @total_size > @count.to_i %>
+  <div class="col-sm-4">
+    <%= erb unique_template(:_paging), locals: { url: "#{root_path}changelogs" } %>
+  </div>
+  <% end %>
+</header>
+<% if @changelogs.any? %>
+<div class="table_container">
+  <form action="<%= root_path %>changelogs/delete_all" method="get">
+    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+  <br/>
+  <table class="table table-striped table-bordered table-hover">
+    <thead>
+      <tr>
+        <th><%= t('Time') %></th>
+        <th><%= t('Digest') %></th>
+        <th><%= t('Script') %></th>
+        <th><%= t('JID') %></th>
+        <th><%= t('Prev JID') %></th>
+        <th><%= t('Message') %></th>
+      </tr>
+    </thead>
+    <tbody>
+    <% @changelogs.each do |changelog| %>
+      <tr  class="changelog-row">
+        <td><%= safe_relative_time(changelog["time"]) %></td>
+        <td><%= changelog["digest"] %></td>
+        <td><%= changelog["script"] %></td>
+        <td><%= changelog["job_id"] %></td>
+        <td><%= changelog["prev_jid"] %></td>
+        <td><%= changelog["message"] %></th>
+      </tr>
+    <% end %>
+    </tbody>
+  </table>
+  <form action="<%= root_path %>changelogs/delete_all" method="get">
+    <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+  </form>
+</div>
+<% end %>

data/lib/sidekiq_unique_jobs/web/views/lock.erb

@@ -0,0 +1,108 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+    <%= t('Lock information') %> <a class="btn btn-default btn-xs" href="<%= root_path %>locks"><%= t('GoBack') %></a>
+    </h3>
+  </div>
+  <div class="col-sm-7 table-responsive">
+    <% if @lock.info.none? %>
+    <h3>No Lock Information Available</h3>
+    <p>To use it turn the following setting on:
+      <code>SidekiqUniqueJobs.config.lock_info = true</code>
+    </p>
+    <% else %>
+    <table class="table table-striped table-bordered table-white table-hover">
+      <caption>Information about lock</caption>
+      <tbody>
+        <tr>
+          <th scope=row><%= t('Worker') %></td>
+          <td><%= @lock.info["worker"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Queue') %></td>
+          <td><%= @lock.info["queue"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Limit') %></td>
+          <td><%= @lock.info["limit"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('TTL') %></td>
+          <td><%= @lock.info["ttl"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Timeout') %></td>
+          <td><%= @lock.info["timeout"] %></td>
+        </tr>
+        <tr>
+          <th scope=row><%= t('Args') %></td>
+          <td>
+            <code class="code-wrap">
+              <!-- We don't want to truncate any job arguments when viewing a single job's status page -->
+              <div class="args-extended"><%= display_lock_args(@lock.info["lock_args"], nil) %></div>
+            </code>
+          </td>
+        </tr>
+      </tbody>
+    </table>
+    <% end %>
+  </div>
+</header>
+<div class="row">
+  <div class="col-sm-6 table-responsive">
+    <table class="table table-striped table-bordered table-hover">
+      <caption>Digest: <strong class="text-muted"><%= @lock.key %></strong></caption>
+      <thead>
+        <tr>
+          <th scope="col"><%= t('Locked JIDs') %></th>
+          <th scope="col"><%= t('Since') %></th>
+          <th scope="col"></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @lock.locked_jids(with_values: true).each do |job_id, time| %>
+        <tr>
+          <td><%= job_id %></td>
+          <td><%= safe_relative_time(time.to_f) %></td>
+          <td>
+            <form action="<%= root_path %>locks/<%= @lock.key %>/jobs/<%= job_id %>/delete" method="get">
+              <%= csrf_tag %>
+              <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Unlock') %>" data-confirm="<%= t('AreYouSure') %>" />
+            </form>
+          </td>
+        </tr>
+        <% end %>
+      </tbody>
+    </table>
+  </div>
+  <div class="col-sm-6 table-responsive">
+    <% if @lock.changelog.count.positive? %>
+    <table class="table table-striped table-bordered table-hover">
+      <caption>Changelogs</caption>
+      <thead>
+        <tr>
+          <th scope="col"><%= t('At') %></th>
+          <th scope="col"><%= t('JID') %></th>
+          <th scope="col"><%= t('Message') %></th>
+          <th scope="col"><%= t('Script') %></th>
+        </tr>
+      </thead>
+      <tbody>
+        <% @lock.changelogs.each do |entry| %>
+        <tr>
+          <td scope="row"><%= safe_relative_time(entry["time"].to_f) %></td>
+          <td><%= entry["job_id"] %></td>
+          <td><%= entry["message"] %></td>
+          <td><%= entry["script"] %></td>
+        </tr>
+        <% end %>
+      </tbody>
+    </table>
+    <% end %>
+  </div>
+</div>
+<form action="<%= root_path %>locks/<%= @lock.key %>/delete" method="get">
+  <%= csrf_tag %>
+  <a class="btn btn-default btn-xs" href="<%= root_path %>locks"><%= t('GoBack') %></a>
+  <input class="btn btn-danger btn-xs flip" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>

data/lib/sidekiq_unique_jobs/web/views/locks.erb

@@ -0,0 +1,54 @@
+<header class="row">
+  <div class="col-sm-5">
+    <h3>
+    <%= t('Locks') %>
+    </h3>
+  </div>
+  <form action="<%= root_path %>locks" class="form form-inline" method="get">
+    <%= csrf_tag %>
+    <input name="filter" class="form-control" type="text" value="<%= @filter %>" />
+    <button class="btn btn-default" type="submit">
+    <%= t('Filter') %>
+    </button>
+  </form>
+  <% if @locks.any? && @total_size > @count %>
+  <div class="col-sm-4">
+    <%= erb unique_template(:_paging), locals: { url: "#{root_path}locks" } %>
+  </div>
+  <% end %>
+</header>
+<% if @locks.any? %>
+<div class="table_container">
+  <table class="table table-striped table-bordered table-hover">
+    <thead>
+      <tr>
+        <th><%= t('Delete') %></th>
+        <th><%= t('Digest') %></th>
+        <th><%= t('Lock') %></th>
+        <th><%= t('Locks') %></th>
+        <th><%= t('Since') %></th>
+      </tr>
+    </thead>
+    <% @locks.each do |lock| %>
+    <tbody>
+      <tr class="lock-row">
+        <td>
+          <form action="<%= root_path %>locks/<%= lock.key %>/delete" method="get">
+            <%= csrf_tag %>
+            <input name="lock" value="<%= h lock.key %>" type="hidden" />
+            <input class="btn btn-danger btn-xs" type="submit" name="delete" value="<%= t('Delete') %>" data-confirm="<%= t('AreYouSure') %>" />
+          </form>
+        </td>
+        <td><a href="<%= root_path %>locks/<%= lock.key %>"><%= lock.key %></a></td>
+        <td><%= lock.info["lock"] %></td>
+        <td><%= lock.locked.count %></td>
+        <td><%= safe_relative_time(lock.created_at) %></td>
+      </tr>
+    </tbody>
+    <% end %>
+</table>
+<form action="<%= root_path %>locks/delete_all" method="get">
+  <input class="btn btn-danger btn-xs" type="submit" name="delete_all" value="<%= t('DeleteAll') %>" data-confirm="<%= t('AreYouSure') %>" />
+</form>
+</div>
+<% end %>