sidekiq 5.2.7 → 6.0.0

data/lib/sidekiq/launcher.rb

@@ -1,19 +1,25 @@
 # frozen_string_literal: true
-require 'sidekiq/manager'
-require 'sidekiq/fetch'
-require 'sidekiq/scheduled'
+
+require "sidekiq/manager"
+require "sidekiq/fetch"
+require "sidekiq/scheduled"
 
 module Sidekiq
-  # The Launcher is a very simple Actor whose job is to
-  # start, monitor and stop the core Actors in Sidekiq.
-  # If any of these actors die, the Sidekiq process exits
-  # immediately.
+  # The Launcher starts the Manager and Poller threads and provides the process heartbeat.
   class Launcher
     include Util
 
-    attr_accessor :manager, :poller, :fetcher
+    STATS_TTL = 5 * 365 * 24 * 60 * 60 # 5 years
 
-    STATS_TTL = 5*365*24*60*60
+    PROCTITLES = [
+      proc { "sidekiq" },
+      proc { Sidekiq::VERSION },
+      proc { |me, data| data["tag"] },
+      proc { |me, data| "[#{Processor::WORKER_STATE.size} of #{data["concurrency"]} busy]" },
+      proc { |me, data| "stopping" if me.stopping? },
+    ]
+
+    attr_accessor :manager, :poller, :fetcher
 
     def initialize(options)
       @manager = Sidekiq::Manager.new(options)
@@ -62,10 +68,30 @@ module Sidekiq
 
     private unless $TESTING
 
+    def start_heartbeat
+      loop do
+        heartbeat
+        sleep 5
+      end
+      Sidekiq.logger.info("Heartbeat stopping...")
+    end
+
+    def clear_heartbeat
+      # Remove record from Redis since we are shutting down.
+      # Note we don't stop the heartbeat thread; if the process
+      # doesn't actually exit, it'll reappear in the Web UI.
+      Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.srem("processes", identity)
+          conn.del("#{identity}:workers")
+        end
+      end
+    rescue
+      # best effort, ignore network errors
+    end
+
     def heartbeat
-      results = Sidekiq::CLI::PROCTITLES.map {|x| x.(self, to_data) }
-      results.compact!
-      $0 = results.join(' ')
+      $0 = PROCTITLES.map { |proc| proc.call(self, to_data) }.compact.join(" ")
 
       ❤
     end
@@ -73,6 +99,7 @@ module Sidekiq
     def ❤
       key = identity
       fails = procd = 0
+
       begin
         fails = Processor::FAILURE.reset
         procd = Processor::PROCESSED.reset
@@ -80,6 +107,7 @@ module Sidekiq
 
         workers_key = "#{key}:workers"
         nowdate = Time.now.utc.strftime("%Y-%m-%d")
+
         Sidekiq.redis do |conn|
           conn.multi do
             conn.incrby("stat:processed", procd)
@@ -97,24 +125,27 @@ module Sidekiq
             conn.expire(workers_key, 60)
           end
         end
+
         fails = procd = 0
 
-        _, exists, _, _, msg = Sidekiq.redis do |conn|
-          conn.multi do
-            conn.sadd('processes', key)
+        _, exists, _, _, msg = Sidekiq.redis { |conn|
+          res = conn.multi {
+            conn.sadd("processes", key)
             conn.exists(key)
-            conn.hmset(key, 'info', to_json, 'busy', curstate.size, 'beat', Time.now.to_f, 'quiet', @done)
+            conn.hmset(key, "info", to_json, "busy", curstate.size, "beat", Time.now.to_f, "quiet", @done)
             conn.expire(key, 60)
             conn.rpop("#{key}-signals")
-          end
-        end
+          }
+
+          res
+        }
 
         # first heartbeat or recovering from an outage and need to reestablish our heartbeat
-        fire_event(:heartbeat) if !exists
+        fire_event(:heartbeat) unless exists
 
         return unless msg
 
-        ::Process.kill(msg, $$)
+        ::Process.kill(msg, ::Process.pid)
       rescue => e
         # ignore all redis/network issues
         logger.error("heartbeat: #{e.message}")
@@ -124,25 +155,17 @@ module Sidekiq
       end
     end
 
-    def start_heartbeat
-      while true
-        heartbeat
-        sleep 5
-      end
-      Sidekiq.logger.info("Heartbeat stopping...")
-    end
-
     def to_data
       @data ||= begin
         {
-          'hostname' => hostname,
-          'started_at' => Time.now.to_f,
-          'pid' => $$,
-          'tag' => @options[:tag] || '',
-          'concurrency' => @options[:concurrency],
-          'queues' => @options[:queues].uniq,
-          'labels' => @options[:labels],
-          'identity' => identity,
+          "hostname" => hostname,
+          "started_at" => Time.now.to_f,
+          "pid" => ::Process.pid,
+          "tag" => @options[:tag] || "",
+          "concurrency" => @options[:concurrency],
+          "queues" => @options[:queues].uniq,
+          "labels" => @options[:labels],
+          "identity" => identity,
         }
       end
     end
@@ -154,20 +177,5 @@ module Sidekiq
         Sidekiq.dump_json(to_data)
       end
     end
-
-    def clear_heartbeat
-      # Remove record from Redis since we are shutting down.
-      # Note we don't stop the heartbeat thread; if the process
-      # doesn't actually exit, it'll reappear in the Web UI.
-      Sidekiq.redis do |conn|
-        conn.pipelined do
-          conn.srem('processes', identity)
-          conn.del("#{identity}:workers")
-        end
-      end
-    rescue
-      # best effort, ignore network errors
-    end
-
   end
 end

data/lib/sidekiq/logger.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "logger"
+require "time"
+
+module Sidekiq
+  class Logger < ::Logger
+    def initialize(*args)
+      super
+
+      self.formatter = Sidekiq.log_formatter
+    end
+
+    def with_context(hash)
+      ctx.merge!(hash)
+      yield
+    ensure
+      hash.keys.each { |key| ctx.delete(key) }
+    end
+
+    def ctx
+      Thread.current[:sidekiq_context] ||= {}
+    end
+
+    module Formatters
+      class Base < ::Logger::Formatter
+        def tid
+          Thread.current["sidekiq_tid"] ||= (Thread.current.object_id ^ ::Process.pid).to_s(36)
+        end
+
+        def ctx
+          Thread.current[:sidekiq_context] ||= {}
+        end
+
+        def format_context
+          " " + ctx.compact.map { |k, v| "#{k}=#{v}" }.join(" ") if ctx.any?
+        end
+      end
+
+      class Pretty < Base
+        def call(severity, time, program_name, message)
+          "#{time.utc.iso8601(3)} pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class WithoutTimestamp < Pretty
+        def call(severity, time, program_name, message)
+          "pid=#{::Process.pid} tid=#{tid}#{format_context} #{severity}: #{message}\n"
+        end
+      end
+
+      class JSON < Base
+        def call(severity, time, program_name, message)
+          hash = {
+            ts: time.utc.iso8601(3),
+            pid: ::Process.pid,
+            tid: tid,
+            lvl: severity,
+            msg: message,
+          }
+          c = ctx
+          hash["ctx"] = c unless c.empty?
+
+          Sidekiq.dump_json(hash) << "\n"
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/manager.rb

@@ -1,12 +1,11 @@
 # frozen_string_literal: true
-require 'sidekiq/util'
-require 'sidekiq/processor'
-require 'sidekiq/fetch'
-require 'thread'
-require 'set'
 
-module Sidekiq
+require "sidekiq/util"
+require "sidekiq/processor"
+require "sidekiq/fetch"
+require "set"
 
+module Sidekiq
   ##
   # The Manager is the central coordination point in Sidekiq, controlling
   # the lifecycle of the Processors.
@@ -27,7 +26,7 @@ module Sidekiq
     attr_reader :workers
     attr_reader :options
 
-    def initialize(options={})
+    def initialize(options = {})
       logger.debug { options.inspect }
       @options = options
       @count = options[:concurrency] || 10
@@ -113,7 +112,7 @@ module Sidekiq
       end
 
       if cleanup.size > 0
-        jobs = cleanup.map {|p| p.job }.compact
+        jobs = cleanup.map { |p| p.job }.compact
 
         logger.warn { "Terminating #{cleanup.size} busy worker threads" }
         logger.warn { "Work still in progress #{jobs.inspect}" }
@@ -132,6 +131,5 @@ module Sidekiq
         processor.kill
       end
     end
-
   end
 end

data/lib/sidekiq/middleware/chain.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 module Sidekiq
   # Middleware is code configured to run before/after
   # a message is processed.  It is patterned after Rack
@@ -106,7 +107,7 @@ module Sidekiq
         i = entries.index { |entry| entry.klass == newklass }
         new_entry = i.nil? ? Entry.new(newklass, *args) : entries.delete_at(i)
         i = entries.index { |entry| entry.klass == oldklass } || entries.count - 1
-        entries.insert(i+1, new_entry)
+        entries.insert(i + 1, new_entry)
       end
 
       def exists?(klass)
@@ -139,7 +140,7 @@ module Sidekiq
 
       def initialize(klass, *args)
         @klass = klass
-        @args  = args
+        @args = args
       end
 
       def make_new

data/lib/sidekiq/middleware/i18n.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 #
 # Simple middleware to save the current locale and restore it when the job executes.
 # Use it by requiring it in your initializer:
@@ -9,19 +10,16 @@ module Sidekiq::Middleware::I18n
   # Get the current locale and store it in the message
   # to be sent to Sidekiq.
   class Client
-    def call(worker_class, msg, queue, redis_pool)
-      msg['locale'] ||= I18n.locale
+    def call(_worker, msg, _queue, _redis)
+      msg["locale"] ||= I18n.locale
       yield
     end
   end
 
   # Pull the msg locale out and set the current thread to use it.
   class Server
-    def call(worker, msg, queue)
-      I18n.locale = msg['locale'] || I18n.default_locale
-      yield
-    ensure
-      I18n.locale = I18n.default_locale
+    def call(_worker, msg, _queue, &block)
+      I18n.with_locale(msg.fetch("locale", I18n.default_locale), &block)
     end
   end
 end

data/lib/sidekiq/monitor.rb

@@ -0,0 +1,148 @@
+#!/usr/bin/env ruby
+
+require "fileutils"
+require "sidekiq/api"
+
+class Sidekiq::Monitor
+  CMD = File.basename($PROGRAM_NAME)
+
+  attr_reader :stage
+
+  def self.print_usage
+    puts "#{CMD} - monitor Sidekiq from the command line."
+    puts
+    puts "Usage: #{CMD} status <section>"
+    puts
+    puts "       <section> (optional) view a specific section of the status output"
+    puts "       Valid sections are: #{Sidekiq::Monitor::Status::VALID_SECTIONS.join(", ")}"
+    puts
+    puts "Set REDIS_URL to the location of your Redis server if not monitoring localhost."
+  end
+
+  class Status
+    VALID_SECTIONS = %w[all version overview processes queues]
+    COL_PAD = 2
+
+    def display(section = nil)
+      section ||= "all"
+      unless VALID_SECTIONS.include? section
+        puts "I don't know how to check the status of '#{section}'!"
+        puts "Try one of these: #{VALID_SECTIONS.join(", ")}"
+        return
+      end
+      send(section)
+    rescue => e
+      puts "Couldn't get status: #{e}"
+    end
+
+    def all
+      version
+      puts
+      overview
+      puts
+      processes
+      puts
+      queues
+    end
+
+    def version
+      puts "Sidekiq #{Sidekiq::VERSION}"
+      puts Time.now
+    end
+
+    def overview
+      puts "---- Overview ----"
+      puts "  Processed: #{delimit stats.processed}"
+      puts "     Failed: #{delimit stats.failed}"
+      puts "       Busy: #{delimit stats.workers_size}"
+      puts "   Enqueued: #{delimit stats.enqueued}"
+      puts "    Retries: #{delimit stats.retry_size}"
+      puts "  Scheduled: #{delimit stats.scheduled_size}"
+      puts "       Dead: #{delimit stats.dead_size}"
+    end
+
+    def processes
+      puts "---- Processes (#{process_set.size}) ----"
+      process_set.each_with_index do |process, index|
+        puts "#{process["identity"]} #{tags_for(process)}"
+        puts "  Started: #{Time.at(process["started_at"])} (#{time_ago(process["started_at"])})"
+        puts "  Threads: #{process["concurrency"]} (#{process["busy"]} busy)"
+        puts "   Queues: #{split_multiline(process["queues"].sort, pad: 11)}"
+        puts "" unless (index + 1) == process_set.size
+      end
+    end
+
+    def queues
+      puts "---- Queues (#{queue_data.size}) ----"
+      columns = {
+        name: [:ljust, (["name"] + queue_data.map(&:name)).map(&:length).max + COL_PAD],
+        size: [:rjust, (["size"] + queue_data.map(&:size)).map(&:length).max + COL_PAD],
+        latency: [:rjust, (["latency"] + queue_data.map(&:latency)).map(&:length).max + COL_PAD],
+      }
+      columns.each { |col, (dir, width)| print col.to_s.upcase.public_send(dir, width) }
+      puts
+      queue_data.each do |q|
+        columns.each do |col, (dir, width)|
+          print q.send(col).public_send(dir, width)
+        end
+        puts
+      end
+    end
+
+    private
+
+    def delimit(number)
+      number.to_s.reverse.scan(/.{1,3}/).join(",").reverse
+    end
+
+    def split_multiline(values, opts = {})
+      return "none" unless values
+      pad = opts[:pad] || 0
+      max_length = opts[:max_length] || (80 - pad)
+      out = []
+      line = ""
+      values.each do |value|
+        if (line.length + value.length) > max_length
+          out << line
+          line = " " * pad
+        end
+        line << value + ", "
+      end
+      out << line[0..-3]
+      out.join("\n")
+    end
+
+    def tags_for(process)
+      tags = [
+        process["tag"],
+        process["labels"],
+        (process["quiet"] == "true" ? "quiet" : nil),
+      ].flatten.compact
+      tags.any? ? "[#{tags.join("] [")}]" : nil
+    end
+
+    def time_ago(timestamp)
+      seconds = Time.now - Time.at(timestamp)
+      return "just now" if seconds < 60
+      return "a minute ago" if seconds < 120
+      return "#{seconds.floor / 60} minutes ago" if seconds < 3600
+      return "an hour ago" if seconds < 7200
+      "#{seconds.floor / 60 / 60} hours ago"
+    end
+
+    QUEUE_STRUCT = Struct.new(:name, :size, :latency)
+    def queue_data
+      @queue_data ||= Sidekiq::Queue.all.map { |q|
+        QUEUE_STRUCT.new(q.name, q.size.to_s, sprintf("%#.2f", q.latency))
+      }
+    end
+
+    def process_set
+      @process_set ||= Sidekiq::ProcessSet.new
+    end
+
+    def stats
+      @stats ||= Sidekiq::Stats.new
+    end
+  end
+end