sidekiq 6.0.0 → 6.0.7

data/lib/sidekiq/scheduled.rb

@@ -14,18 +14,19 @@ module Sidekiq
         # Just check Redis for the set of jobs with a timestamp before now.
         Sidekiq.redis do |conn|
           sorted_sets.each do |sorted_set|
-            # Get the next item in the queue if it's score (time to execute) is <= now.
-            # We need to go through the list one at a time to reduce the risk of something
-            # going wrong between the time jobs are popped from the scheduled queue and when
-            # they are pushed onto a work queue and losing the jobs.
-            while (job = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 1]).first)
-
-              # Pop item off the queue and add it to the work queue. If the job can't be popped from
-              # the queue, it's because another process already popped it so we can move on to the
-              # next one.
-              if conn.zrem(sorted_set, job)
-                Sidekiq::Client.push(Sidekiq.load_json(job))
-                Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+            # Get next items in the queue with scores (time to execute) <= now.
+            until (jobs = conn.zrangebyscore(sorted_set, "-inf", now, limit: [0, 100])).empty?
+              # We need to go through the list one at a time to reduce the risk of something
+              # going wrong between the time jobs are popped from the scheduled queue and when
+              # they are pushed onto a work queue and losing the jobs.
+              jobs.each do |job|
+                # Pop item off the queue and add it to the work queue. If the job can't be popped from
+                # the queue, it's because another process already popped it so we can move on to the
+                # next one.
+                if conn.zrem(sorted_set, job)
+                  Sidekiq::Client.push(Sidekiq.load_json(job))
+                  Sidekiq.logger.debug { "enqueued #{sorted_set}: #{job}" }
+                end
               end
             end
           end

data/lib/sidekiq/sd_notify.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+# The MIT License
+#
+# Copyright (c) 2017, 2018, 2019, 2020 Agis Anastasopoulos
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+# This is a copy of https://github.com/agis/ruby-sdnotify as of commit a7d52ee
+# The only changes made was "rehoming" it within the Sidekiq module to avoid
+# namespace collisions and applying standard's code formatting style.
+
+require "socket"
+
+# SdNotify is a pure-Ruby implementation of sd_notify(3). It can be used to
+# notify systemd about state changes. Methods of this package are no-op on
+# non-systemd systems (eg. Darwin).
+#
+# The API maps closely to the original implementation of sd_notify(3),
+# therefore be sure to check the official man pages prior to using SdNotify.
+#
+# @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+module Sidekiq
+  module SdNotify
+    # Exception raised when there's an error writing to the notification socket
+    class NotifyError < RuntimeError; end
+
+    READY = "READY=1"
+    RELOADING = "RELOADING=1"
+    STOPPING = "STOPPING=1"
+    STATUS = "STATUS="
+    ERRNO = "ERRNO="
+    MAINPID = "MAINPID="
+    WATCHDOG = "WATCHDOG=1"
+    FDSTORE = "FDSTORE=1"
+
+    def self.ready(unset_env = false)
+      notify(READY, unset_env)
+    end
+
+    def self.reloading(unset_env = false)
+      notify(RELOADING, unset_env)
+    end
+
+    def self.stopping(unset_env = false)
+      notify(STOPPING, unset_env)
+    end
+
+    # @param status [String] a custom status string that describes the current
+    #   state of the service
+    def self.status(status, unset_env = false)
+      notify("#{STATUS}#{status}", unset_env)
+    end
+
+    # @param errno [Integer]
+    def self.errno(errno, unset_env = false)
+      notify("#{ERRNO}#{errno}", unset_env)
+    end
+
+    # @param pid [Integer]
+    def self.mainpid(pid, unset_env = false)
+      notify("#{MAINPID}#{pid}", unset_env)
+    end
+
+    def self.watchdog(unset_env = false)
+      notify(WATCHDOG, unset_env)
+    end
+
+    def self.fdstore(unset_env = false)
+      notify(FDSTORE, unset_env)
+    end
+
+    # @param [Boolean] true if the service manager expects watchdog keep-alive
+    #   notification messages to be sent from this process.
+    #
+    # If the $WATCHDOG_USEC environment variable is set,
+    # and the $WATCHDOG_PID variable is unset or set to the PID of the current
+    # process
+    #
+    # @note Unlike sd_watchdog_enabled(3), this method does not mutate the
+    #   environment.
+    def self.watchdog?
+      wd_usec = ENV["WATCHDOG_USEC"]
+      wd_pid = ENV["WATCHDOG_PID"]
+
+      return false unless wd_usec
+
+      begin
+        wd_usec = Integer(wd_usec)
+      rescue
+        return false
+      end
+
+      return false if wd_usec <= 0
+      return true if !wd_pid || wd_pid == $$.to_s
+
+      false
+    end
+
+    # Notify systemd with the provided state, via the notification socket, if
+    # any.
+    #
+    # Generally this method will be used indirectly through the other methods
+    # of the library.
+    #
+    # @param state [String]
+    # @param unset_env [Boolean]
+    #
+    # @return [Fixnum, nil] the number of bytes written to the notification
+    #   socket or nil if there was no socket to report to (eg. the program wasn't
+    #   started by systemd)
+    #
+    # @raise [NotifyError] if there was an error communicating with the systemd
+    #   socket
+    #
+    # @see https://www.freedesktop.org/software/systemd/man/sd_notify.html
+    def self.notify(state, unset_env = false)
+      sock = ENV["NOTIFY_SOCKET"]
+
+      return nil unless sock
+
+      ENV.delete("NOTIFY_SOCKET") if unset_env
+
+      begin
+        Addrinfo.unix(sock, :DGRAM).connect do |s|
+          s.close_on_exec = true
+          s.write(state)
+        end
+      rescue => e
+        raise NotifyError, "#{e.class}: #{e.message}", e.backtrace
+      end
+    end
+  end
+end

data/lib/sidekiq/systemd.rb

@@ -0,0 +1,24 @@
+#
+# Sidekiq's systemd integration allows Sidekiq to inform systemd:
+#  1. when it has successfully started
+#  2. when it is starting shutdown
+#  3. periodically for a liveness check with a watchdog thread
+#
+module Sidekiq
+  def self.start_watchdog
+    usec = Integer(ENV["WATCHDOG_USEC"])
+    return Sidekiq.logger.error("systemd Watchdog too fast: " + usec) if usec < 1_000_000
+
+    sec_f = usec / 1_000_000.0
+    # "It is recommended that a daemon sends a keep-alive notification message
+    # to the service manager every half of the time returned here."
+    ping_f = sec_f / 2
+    Sidekiq.logger.info "Pinging systemd watchdog every #{ping_f.round(1)} sec"
+    Thread.new do
+      loop do
+        sleep ping_f
+        Sidekiq::SdNotify.watchdog
+      end
+    end
+  end
+end

data/lib/sidekiq/testing.rb

@@ -323,6 +323,18 @@ module Sidekiq
       end
     end
   end
+
+  module TestingExtensions
+    def jobs_for(klass)
+      jobs.select do |job|
+        marshalled = job["args"][0]
+        marshalled.index(klass.to_s) && YAML.load(marshalled)[0] == klass
+      end
+    end
+  end
+
+  Sidekiq::Extensions::DelayedMailer.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedMailer)
+  Sidekiq::Extensions::DelayedModel.extend(TestingExtensions) if defined?(Sidekiq::Extensions::DelayedModel)
 end
 
 if defined?(::Rails) && Rails.respond_to?(:env) && !Rails.env.test?

data/lib/sidekiq/util.rb

@@ -11,8 +11,6 @@ module Sidekiq
   module Util
     include ExceptionHandler
 
-    EXPIRY = 60 * 60 * 24
-
     def watchdog(last_words)
       yield
     rescue Exception => ex

data/lib/sidekiq/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Sidekiq
-  VERSION = "6.0.0"
+  VERSION = "6.0.7"
 end

data/lib/sidekiq/web/application.rb

@@ -5,7 +5,6 @@ module Sidekiq
     extend WebRouter
 
     CONTENT_LENGTH = "Content-Length"
-    CONTENT_TYPE = "Content-Type"
     REDIS_KEYS = %w[redis_version uptime_in_days connected_clients used_memory_human used_memory_peak_human]
     CSP_HEADER = [
       "default-src 'self' https: http:",
@@ -20,7 +19,7 @@ module Sidekiq
       "script-src 'self' https: http: 'unsafe-inline'",
       "style-src 'self' https: http: 'unsafe-inline'",
       "worker-src 'self'",
-      "base-uri 'self'",
+      "base-uri 'self'"
     ].join("; ").freeze
 
     def initialize(klass)
@@ -84,14 +83,22 @@ module Sidekiq
 
       @count = (params["count"] || 25).to_i
       @queue = Sidekiq::Queue.new(@name)
-      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count)
+      (@current_page, @total_size, @messages) = page("queue:#{@name}", params["page"], @count, reverse: params["direction"] == "asc")
       @messages = @messages.map { |msg| Sidekiq::Job.new(msg, @name) }
 
       erb(:queue)
     end
 
     post "/queues/:name" do
-      Sidekiq::Queue.new(route_params[:name]).clear
+      queue = Sidekiq::Queue.new(route_params[:name])
+
+      if Sidekiq.pro? && params["pause"]
+        queue.pause!
+      elsif Sidekiq.pro? && params["unpause"]
+        queue.unpause!
+      else
+        queue.clear
+      end
 
       redirect "#{root_path}queues"
     end
@@ -268,7 +275,7 @@ module Sidekiq
           scheduled: sidekiq_stats.scheduled_size,
           retries: sidekiq_stats.retry_size,
           dead: sidekiq_stats.dead_size,
-          default_latency: sidekiq_stats.default_queue_latency,
+          default_latency: sidekiq_stats.default_queue_latency
         },
         redis: redis_stats,
         server_utc_time: server_utc_time
@@ -283,36 +290,30 @@ module Sidekiq
       action = self.class.match(env)
       return [404, {"Content-Type" => "text/plain", "X-Cascade" => "pass"}, ["Not Found"]] unless action
 
-      resp = catch(:halt) {
-        app = @klass
+      app = @klass
+      resp = catch(:halt) do # rubocop:disable Standard/SemanticBlocks
         self.class.run_befores(app, action)
-        begin
-          resp = action.instance_exec env, &action.block
-        ensure
-          self.class.run_afters(app, action)
-        end
-
-        resp
-      }
+        action.instance_exec env, &action.block
+      ensure
+        self.class.run_afters(app, action)
+      end
 
       resp = case resp
       when Array
+        # redirects go here
         resp
       else
+        # rendered content goes here
         headers = {
           "Content-Type" => "text/html",
           "Cache-Control" => "no-cache",
           "Content-Language" => action.locale,
-          "Content-Security-Policy" => CSP_HEADER,
+          "Content-Security-Policy" => CSP_HEADER
         }
-
+        # we'll let Rack calculate Content-Length for us.
         [200, headers, [resp]]
       end
 
-      resp[1] = resp[1].dup
-
-      resp[1][CONTENT_LENGTH] = resp[2].inject(0) { |l, p| l + p.bytesize }.to_s
-
       resp
     end
 

data/lib/sidekiq/web/helpers.rb

@@ -65,7 +65,10 @@ module Sidekiq
 
     def poll_path
       if current_path != "" && params["poll"]
-        root_path + current_path
+        path = root_path + current_path
+        query_string = to_query_string(params.slice(*params.keys - %w[page poll]))
+        path += "?#{query_string}" unless query_string.empty?
+        path
       else
         ""
       end
@@ -112,6 +115,13 @@ module Sidekiq
       end
     end
 
+    # within is used by Sidekiq Pro
+    def display_tags(job, within = nil)
+      job.tags.map { |tag|
+        "<span class='jobtag label label-info'>#{::Rack::Utils.escape_html(tag)}</span>"
+      }.join(" ")
+    end
+
     # mperham/sidekiq#3243
     def unfiltered?
       yield unless env["PATH_INFO"].start_with?("/filter/")
@@ -130,6 +140,10 @@ module Sidekiq
       end
     end
 
+    def sort_direction_label
+      params[:direction] == "asc" ? "&uarr;" : "&darr;"
+    end
+
     def workers
       @workers ||= Sidekiq::Workers.new
     end
@@ -142,12 +156,6 @@ module Sidekiq
       @stats ||= Sidekiq::Stats.new
     end
 
-    def retries_with_score(score)
-      Sidekiq.redis { |conn|
-        conn.zrangebyscore("retry", score, score)
-      }.map { |msg| Sidekiq.load_json(msg) }
-    end
-
     def redis_connection
       Sidekiq.redis do |conn|
         c = conn.connection
@@ -189,7 +197,7 @@ module Sidekiq
       [score.to_f, jid]
     end
 
-    SAFE_QPARAMS = %w[page poll]
+    SAFE_QPARAMS = %w[page poll direction]
 
     # Merge options with current params, filter safe params, and stringify to query string
     def qparams(options)
@@ -198,7 +206,11 @@ module Sidekiq
         options[key.to_s] = options.delete(key)
       end
 
-      params.merge(options).map { |key, value|
+      to_query_string(params.merge(options))
+    end
+
+    def to_query_string(params)
+      params.map { |key, value|
         SAFE_QPARAMS.include?(key) ? "#{key}=#{CGI.escape(value.to_s)}" : next
       }.compact.join("&")
     end
@@ -238,7 +250,7 @@ module Sidekiq
       queue class args retry_count retried_at failed_at
       jid error_message error_class backtrace
       error_backtrace enqueued_at retry wrapped
-      created_at
+      created_at tags
     ])
 
     def retry_extra_items(retry_job)
@@ -283,7 +295,7 @@ module Sidekiq
     end
 
     def environment_title_prefix
-      environment = Sidekiq.options[:environment] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
+      environment = Sidekiq.options[:environment] || ENV["APP_ENV"] || ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
 
       "[#{environment.upcase}] " unless environment == "production"
     end

data/lib/sidekiq/web/router.rb

@@ -94,9 +94,7 @@ module Sidekiq
         {} if path == matcher
       else
         path_match = path.match(matcher)
-        if path_match
-          Hash[path_match.names.map(&:to_sym).zip(path_match.captures)]
-        end
+        path_match&.named_captures&.transform_keys(&:to_sym)
       end
     end
   end

data/lib/sidekiq/web.rb

@@ -31,7 +31,7 @@ module Sidekiq
       "Queues" => "queues",
       "Retries" => "retries",
       "Scheduled" => "scheduled",
-      "Dead" => "morgue",
+      "Dead" => "morgue"
     }
 
     class << self

data/lib/sidekiq/worker.rb

@@ -48,8 +48,8 @@ module Sidekiq
         # In practice, any option is allowed.  This is the main mechanism to configure the
         # options for a specific job.
         def sidekiq_options(opts = {})
-          opts = Hash[opts.map { |k, v| [k.to_s, v] }] # stringify
-          self.sidekiq_options_hash = get_sidekiq_options.merge(Hash[opts.map { |k, v| [k.to_s, v] }])
+          opts = opts.transform_keys(&:to_s) # stringify
+          self.sidekiq_options_hash = get_sidekiq_options.merge(opts)
         end
 
         def sidekiq_retry_in(&block)
@@ -171,9 +171,9 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        payload = @opts.merge("class" => @klass, "args" => args, "at" => ts)
+        payload = @opts.merge("class" => @klass, "args" => args)
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        payload.delete("at") if ts <= now
+        payload["at"] = ts if ts > now
         @klass.client_push(payload)
       end
       alias_method :perform_at, :perform_in
@@ -207,10 +207,10 @@ module Sidekiq
         now = Time.now.to_f
         ts = (int < 1_000_000_000 ? now + int : int)
 
-        item = {"class" => self, "args" => args, "at" => ts}
+        item = {"class" => self, "args" => args}
 
         # Optimization to enqueue something now that is scheduled to go out now or in the past
-        item.delete("at") if ts <= now
+        item["at"] = ts if ts > now
 
         client_push(item)
       end

data/lib/sidekiq.rb

@@ -30,16 +30,16 @@ module Sidekiq
       startup: [],
       quiet: [],
       shutdown: [],
-      heartbeat: [],
+      heartbeat: []
     },
     dead_max_jobs: 10_000,
     dead_timeout_in_seconds: 180 * 24 * 60 * 60, # 6 months
-    reloader: proc { |&block| block.call },
+    reloader: proc { |&block| block.call }
   }
 
   DEFAULT_WORKER_OPTIONS = {
     "retry" => true,
-    "queue" => "default",
+    "queue" => "default"
   }
 
   FAKE_INFO = {
@@ -47,7 +47,7 @@ module Sidekiq
     "uptime_in_days" => "9999",
     "connected_clients" => "9999",
     "used_memory_human" => "9P",
-    "used_memory_peak_human" => "9P",
+    "used_memory_peak_human" => "9P"
   }
 
   def self.❨╯°□°❩╯︵┻━┻
@@ -154,7 +154,7 @@ module Sidekiq
 
   def self.default_worker_options=(hash)
     # stringify
-    @default_worker_options = default_worker_options.merge(Hash[hash.map { |k, v| [k.to_s, v] }])
+    @default_worker_options = default_worker_options.merge(hash.transform_keys(&:to_s))
   end
 
   def self.default_worker_options
@@ -192,6 +192,7 @@ module Sidekiq
 
   def self.log_formatter=(log_formatter)
     @log_formatter = log_formatter
+    logger.formatter = log_formatter
   end
 
   def self.logger
@@ -199,9 +200,20 @@ module Sidekiq
   end
 
   def self.logger=(logger)
+    if logger.nil?
+      self.logger.level = Logger::FATAL
+      return self.logger
+    end
+
+    logger.extend(Sidekiq::LoggingUtils)
+
     @logger = logger
   end
 
+  def self.pro?
+    defined?(Sidekiq::Pro)
+  end
+
   # How frequently Redis should be checked by a random Sidekiq process for
   # scheduled and retriable jobs. Each individual process will take turns by
   # waiting some multiple of this value.

data/sidekiq.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
   gem.homepage = "http://sidekiq.org"
   gem.license = "LGPL-3.0"
 
-  gem.executables = ["sidekiq"]
+  gem.executables = ["sidekiq", "sidekiqmon"]
   gem.files = `git ls-files | grep -Ev '^(test|myapp|examples)'`.split("\n")
   gem.name = "sidekiq"
   gem.version = Sidekiq::VERSION
@@ -16,6 +16,6 @@ Gem::Specification.new do |gem|
 
   gem.add_dependency "redis", ">= 4.1.0"
   gem.add_dependency "connection_pool", ">= 2.2.2"
-  gem.add_dependency "rack", ">= 2.0.0"
+  gem.add_dependency "rack", "~> 2.0"
   gem.add_dependency "rack-protection", ">= 2.0.0"
 end

data/web/assets/javascripts/application.js

@@ -19,7 +19,8 @@ Sidekiq = {};
 $(function() {
   var pollpath = $('body').data('poll-path');
   if (pollpath != "") {
-    updatePage(pollpath);
+    var ti = parseInt(localStorage.timeInterval) || 2000;
+    setTimeout(function(){updatePage(pollpath)}, ti);
   }
 
   $(document).on('click', '.check_all', function() {
@@ -55,26 +56,28 @@ function updateFuzzyTimes(locale) {
 }
 
 function updatePage(url) {
-  setInterval(function () {
-    $.ajax({
-      url: url,
-      dataType: 'html'
-    }).done(function (data) {
-      $data = $(data)
-
-      var $page = $data.filter('#page')
-      $('#page').replaceWith($page)
-
-      var $header_status = $data.find('.status')
-      $('.status').replaceWith($header_status)
-
-      updateFuzzyTimes($('body').data('locale'));
-    })
-  }, parseInt(localStorage.timeInterval) || 2000);
+  $.ajax({
+    url: url,
+    dataType: 'html'
+  }).done(function(data) {
+    $data = $(data)
+
+    var $page = $data.filter('#page')
+    $('#page').replaceWith($page)
+
+    var $header_status = $data.find('.status')
+    $('.status').replaceWith($header_status)
+
+    updateFuzzyTimes($('body').data('locale'));
+
+    var ti = parseInt(localStorage.timeInterval) || 2000;
+    setTimeout(function(){updatePage(url)}, ti)
+  }).fail(function() {
+    var ti = parseInt(localStorage.timeInterval) || 2000;
+    setTimeout(function(){updatePage(url)}, ti)
+  })
 }
 
-
-
 $(function() {
   'use strict';
 

data/web/assets/javascripts/dashboard.js

@@ -27,7 +27,7 @@ var realtimeGraph = function(updatePath) {
     renderer: 'line',
     interpolation: 'linear',
 
-    series: new Rickshaw.Series.FixedDuration([{ name: graphElement.dataset.failedLabel, color: '#B1003E' }, { name: graphElement.dataset.processedLabel, color: '#006f68' }], undefined, {
+    series: new Rickshaw.Series.FixedDuration([{ name: graphElement.dataset.failedLabel, color: '#af0014' }, { name: graphElement.dataset.processedLabel, color: '#006f68' }], undefined, {
       timeInterval: timeInterval,
       maxDataPoints: 100,
     })
@@ -125,7 +125,7 @@ var historyGraph = function() {
     interpolation: 'linear',
     series: [
       {
-        color: "#B1003E",
+        color: "#af0014",
         data: failed,
         name: graphElement.dataset.failedLabel
       }, {