sidekiq_utils 1.0.0

data/lib/sidekiq_utils/latency_alert.rb

@@ -0,0 +1,97 @@
+require 'yaml'
+
+module SidekiqUtils
+  class LatencyAlert
+    REDIS_KEY = "sidekiq_queue_latency_alert"
+
+    class << self
+      def check!
+        alerts = {}
+        Sidekiq::Queue.all.each do |queue|
+          threshold = config['alert_thresholds'][queue.name]
+          next if threshold == :disabled
+          threshold ||= config['alert_thresholds']['default'] || 10.minutes
+          if (latency = queue.latency) > threshold
+            alerts[queue.name] = latency
+          end
+        end
+
+        if alerts.blank?
+          if should_alert_back_to_normal?
+            Sidekiq.redis { |r| r.del(REDIS_KEY) }
+            slack_alert("All queues under their thresholds.")
+          end
+          return false
+        end
+
+        alert_message = ["Sidekiq queue latency over threshold:"]
+        alerts.each do |queue, latency|
+          alert_message << "Queue #{queue} is #{formatted_latency(latency)} behind"
+        end
+        alert_message = alert_message.join("\n")
+        if should_alert_again?(alert_message)
+          slack_alert(alert_message)
+        end
+
+        true
+      end
+
+      def config
+        @config ||= (YAML.load(ERB.new(
+          File.read('config/sidekiq_utils.yml')).result) || {})
+      end
+
+      private
+      def formatted_latency(latency)
+        latency_days = (latency.to_f / 1.day).floor
+        if latency < 1.day
+          formatted_latency =
+            ActionController::Base.helpers.distance_of_time_in_words(latency)
+        else
+          formatted_latency = "#{latency_days} #{"day".pluralize(latency_days)}"
+        end
+        if latency > 1.day
+          latency_in_day = latency - latency_days * 1.day
+          if latency_in_day >= 45.minutes
+            formatted_latency += " and " +
+              ActionController::Base.helpers.
+                distance_of_time_in_words(latency_in_day)
+          end
+        end
+        formatted_latency
+      end
+
+      def slack_alert(alert_message)
+        Array.wrap(config['channels_to_alert']).each do |slack_name|
+          Slack.send_message(
+            slack_name, alert_message,
+            icon: ':alarm_clock:', username: 'Sidekiq alerts')
+        end
+      end
+
+      def should_alert_back_to_normal?
+        Sidekiq.redis do |redis|
+          redis.get(REDIS_KEY).present?
+        end
+      end
+
+      def should_alert_again?(message)
+        Sidekiq.redis do |redis|
+          last_alert = redis.get(REDIS_KEY)
+          last_alert = JSON.parse(last_alert) if last_alert
+          if !last_alert ||
+              last_alert['message_hash'] != Digest::SHA1.hexdigest(message) ||
+              last_alert['time'] < (config['repeat_alert_every'] || 60).to_i.minutes.ago.to_i
+            redis.set(REDIS_KEY, {
+              'message_hash' => Digest::SHA1.hexdigest(message),
+              'time' => Time.now.to_i,
+            }.to_json)
+            true
+          else
+            false
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/client/additional_serialization.rb

@@ -0,0 +1,20 @@
+module SidekiqUtils
+  module Middleware
+    module Client
+      class AdditionalSerialization
+        def call(worker_class, job, queue, redis_pool)
+          if job['class'] == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+            # this is handled in ActiveJob, as it would otherwise raise an
+            # exception before it even gets here
+            return yield
+          end
+
+          job['args'] = job['args'].map do |arg|
+            ::SidekiqUtils::AdditionalSerialization.wrap_argument(arg)
+          end
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/client/deprioritize.rb

@@ -0,0 +1,14 @@
+module SidekiqUtils
+  module Middleware
+    module Client
+      class Deprioritize
+        def call(worker_class, job, queue, redis_pool)
+          if Thread.current[:deprioritize_worker_classes].to_a.include?(worker_class)
+            job['queue'] = 'low'
+          end
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/client/job_counter.rb

@@ -0,0 +1,17 @@
+module SidekiqUtils
+  module Middleware
+    module Client
+      class JobCounter
+        def call(worker_class, job, queue, redis_pool)
+          unless job['at']
+            # don't count when jobs get put on the scheduled set, because
+            # otherwise we'll double-count them when they get popped and moved
+            # to a work queue.
+            SidekiqUtils::JobCounter.increment(job)
+          end
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/server/additional_serialization.rb

@@ -0,0 +1,20 @@
+module SidekiqUtils
+  module Middleware
+    module Server
+      class AdditionalSerialization
+        def call(worker, job, queue)
+          if job['class'] == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+            # this is handled in ActiveJob, as it would otherwise raise an
+            # exception before it even gets here
+            return yield
+          end
+
+          job['args'] = job['args'].map do |arg|
+            ::SidekiqUtils::AdditionalSerialization.unwrap_argument(arg)
+          end
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/server/find_optional.rb

@@ -0,0 +1,20 @@
+module SidekiqUtils
+  module Middleware
+    module Server
+      class FindOptional
+        def call(worker, job, queue)
+          begin
+            yield
+          rescue SidekiqUtils::FindOptional::NotFoundError
+            if queue == 'retry_once'
+              # do nothing; this is already the retry and it failed again
+            else
+              worker.class.set(queue: :retry_once).
+                perform_in(30.seconds, *job['args'])
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/server/job_counter.rb

@@ -0,0 +1,15 @@
+module SidekiqUtils
+  module Middleware
+    module Server
+      class JobCounter
+        def call(worker, job, queue)
+          # we decrement here whether the job succeeds or not, because
+          # re-enqueuing from the retry queue triggers the client middleware
+          # and thus another increment even in the case of an error
+          SidekiqUtils::JobCounter.decrement(job)
+          yield
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/server/memory_monitor.rb

@@ -0,0 +1,46 @@
+require 'objspace'
+
+module SidekiqUtils
+  module Middleware
+    module Server
+      class MemoryMonitor
+        def call(worker, job, queue)
+          return yield unless Sidekiq.options[:concurrency] == 1
+
+          objects_before = count_allocated_objects
+          memory_before = get_allocated_memory
+
+          GC.start(full_mark: true)
+          GC.disable
+          begin
+            yield
+          ensure
+            GC.enable
+            GC.start(full_mark: true)
+            objects_after = count_allocated_objects
+            memory_after = get_allocated_memory
+
+            object_growth = objects_after - objects_before
+            SidekiqUtils::RedisMonitorStorage.store(
+              'sidekiq_memory', 'object', job, object_growth)
+            Sidekiq.logger.info("Object growth: #{object_growth}")
+
+            memory_growth = memory_after - memory_before
+            SidekiqUtils::RedisMonitorStorage.store(
+              'sidekiq_memory', 'memory', job, memory_growth)
+            Sidekiq.logger.info("Memory growth: #{memory_growth}")
+          end
+        end
+
+        private
+        def count_allocated_objects
+          ObjectSpace.each_object.inject(0) {|count, obj| count + 1 }
+        end
+
+        def get_allocated_memory
+          ObjectSpace.memsize_of_all
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/middleware/server/throughput_monitor.rb

@@ -0,0 +1,25 @@
+module SidekiqUtils
+  module Middleware
+    module Server
+      class ThroughputMonitor
+        def call(worker, job, queue)
+          start = Time.now
+          begin
+            yield
+          ensure
+            elapsed = Time.now - start
+            elapsed_ms = (elapsed * 1_000).round
+
+            SidekiqUtils::RedisMonitorStorage.store(
+              'sidekiq_elapsed', 'elapsed', job, elapsed_ms)
+            Sidekiq.redis do |redis|
+              redis.hset('sidekiq_last_run',
+                         SidekiqUtils::RedisMonitorStorage.job_prefix(job),
+                         Time.now.to_i)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/redis_monitor_storage.rb

@@ -0,0 +1,81 @@
+module SidekiqUtils
+  module RedisMonitorStorage
+    class << self
+      def add_first_argument_to_job_key(*klasses)
+        @first_argument_to_job_key_for ||= []
+        @first_argument_to_job_key_for |= klasses
+      end
+
+      def store(key, prefix, job, value)
+        Sidekiq.redis do |redis|
+          redis.multi do
+            redis.hincrby(key, full_prefix(job, prefix, 'sum'), value)
+            redis.hincrby(key, full_prefix(job, prefix, 'count'), 1)
+          end
+        end
+      end
+
+      def retrieve(top_level_key, prefix)
+        data = {}
+        Sidekiq.redis {|r| r.hgetall(top_level_key) }.each do |key, value|
+          (job, prefix_type, date, value_type) = key.split('||')
+          next unless prefix_type == prefix
+
+          if Date.parse(date) < 1.week.ago
+            # expired data, get rid of it
+            Sidekiq.redis {|r| r.hdel(top_level_key, key) }
+          else
+            data[job] ||= { 'sum' => 0, 'count' => 0 }
+            data[job][value_type] += value.to_i
+          end
+        end
+
+        data.each do |job, values|
+          values['average'] = (values['sum'].to_f / values['count'].to_i).round
+        end
+        data
+      end
+
+      def full_prefix(job, prefix = nil, last_prefix = nil)
+        job_prefix = job_prefix(job)
+        full_prefix = [job_prefix, prefix, Date.today.to_s(:medium), last_prefix]
+        full_prefix.compact.join('||')
+      end
+
+      def job_prefix(job, unwrap_arguments: false)
+        arguments = arguments(job)
+        if unwrap_arguments
+          arguments = arguments.
+            map {|arg| SidekiqUtils::AdditionalSerialization.unwrap_argument(arg) }
+        end
+
+        if active_job?(job)
+          job_prefix = job['wrapped']
+        else
+          job_prefix = job['class']
+        end
+
+        case job_prefix
+        when 'ActionMailer::DeliveryJob'
+          job_prefix += "[#{arguments[0..1].join('#')}]"
+        when *(@first_argument_to_job_key_for.to_a)
+          job_prefix += "[#{arguments[0]}]"
+        end
+
+        job_prefix
+      end
+
+      def arguments(job)
+        if active_job?(job)
+          job['args'].first['arguments']
+        else
+          job['args']
+        end
+      end
+
+      def active_job?(job)
+        job['class'] == "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper"
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/views/job_counts.erb

@@ -0,0 +1,34 @@
+<header class='row'>
+  <div class='col-sm-5'>
+    <h3>Enqueued Job Counts</h3>
+  </div>
+</header>
+
+<p>
+This data may not be 100% accurate if jobs haven been manually deleted
+or queues have been manually cleared.
+</p>
+<table class='table table-striped table-bordered table-white'>
+  <thead>
+    <th>Queue</th>
+    <th>Job</th>
+    <th>Count</th>
+    <th>Approx. sequential run time (days:hours:minutes)</th>
+  </thead>
+  <tbody>
+    <% @counts.each do |values| %>
+      <tr>
+        <td><%= values[:queue] %></td>
+        <td><%= values[:job] %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_delimited(values[:count]) %></td>
+        <td>
+          <% if values[:avg_runtime] %>
+            <%= values[:runtime_day] %>:<%= values[:runtime_hour].to_s.rjust(2, '0') %>:<%= values[:runtime_min].to_s.rjust(2, '0') %>
+          <% else %>
+            -/-
+          <% end %>
+        </td>
+      </tr>
+    <% end %>
+  </tbody>
+</table>

data/lib/sidekiq_utils/views/memory.erb

@@ -0,0 +1,33 @@
+<header class='row'>
+  <div class='col-sm-5'>
+    <h3>Memory</h3>
+  </div>
+</header>
+
+<p>
+Data from the trailing seven days. Only a small percentage of jobs get
+profiled. The "absolute" numbers refer only to those jobs that did get
+profiled.
+</p>
+<table class='table table-striped table-bordered table-white'>
+  <thead>
+    <tr>
+      <th>Job</th>
+      <th>Memory growth (absolute)</th>
+      <th>Memory growth (per job)</th>
+      <th>Object growth (absolute)</th>
+      <th>Object growth (per job)</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @memory.each do |(job, avg_memory, avg_object, abs_memory, abs_object)| %>
+      <tr>
+        <td><%= job %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_human_size(abs_memory) %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_human_size(avg_memory) %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_delimited(abs_object) %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_delimited(avg_object) %></td>
+      </tr>
+    <% end %>
+  </tbody>
+</table

data/lib/sidekiq_utils/views/throughput.erb

@@ -0,0 +1,27 @@
+<header class='row'>
+  <div class='col-sm-5'>
+    <h3>Throughput</h3>
+  </div>
+</header>
+
+<p>Data from the trailing seven days.</p>
+<table class='table table-striped table-bordered table-white'>
+  <thead>
+    <tr>
+      <th>Job</th>
+      <th>Avg. execution time (minute:second.millisecond)</th>
+      <th>Total processed</th>
+      <th>Last run at</th>
+    </tr>
+  </thead>
+  <tbody>
+    <% @throughput.each do |(job, avg_execution_time_ms, total_processed, last_run_at)| %>
+      <tr>
+        <td><%= job %></td>
+        <td><%= Time.at(avg_execution_time_ms.to_f / 1_000).utc.strftime("%M:%S.%L") %></td>
+        <td><%= ActiveSupport::NumberHelper.number_to_delimited(total_processed) %></td>
+        <td><%= last_run_at %></td>
+      </tr>
+    <% end %>
+  <tbody>
+</table

data/lib/sidekiq_utils/web_extensions/job_counter.rb

@@ -0,0 +1,35 @@
+module SidekiqUtils
+  module WebExtensions
+    module JobCounter
+      def self.registered(app)
+        view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+        require 'active_support/number_helper'
+        app.get("/job_counts") do
+          @throughput = SidekiqUtils::RedisMonitorStorage.
+              retrieve('sidekiq_elapsed', 'elapsed')
+          @counts = []
+          SidekiqUtils::JobCounter.counts.each do |queue, job_counts|
+            job_counts.each do |job, count|
+              values = { queue: queue, job: job, count: count }
+              if (values[:avg_runtime] = @throughput[job].try!(:[], 'average'))
+                execution_time = values[:count] * values[:avg_runtime].to_f / 1_000
+                values[:runtime_day] = days =
+                  (execution_time / 1.day).floor
+                values[:runtime_hour] = hours =
+                  ((execution_time - days.days) / 1.hour).floor
+                values[:runtime_min] = (
+                  (execution_time - days.days - hours.hours) / 1.minute
+                ).round
+              end
+              @counts << values
+            end
+          end
+          @counts.sort_by! {|x| -1 * x[:count] }
+
+          render(:erb, File.read(File.join(view_path, "job_counts.erb")))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/web_extensions/memory_monitor.rb

@@ -0,0 +1,26 @@
+module SidekiqUtils
+  module WebExtensions
+    module MemoryMonitor
+      def self.registered(app)
+        view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+        require 'active_support/number_helper'
+        app.get("/memory") do
+          memory = SidekiqUtils::RedisMonitorStorage.retrieve('sidekiq_memory', 'memory')
+          object = SidekiqUtils::RedisMonitorStorage.retrieve('sidekiq_memory', 'object')
+
+          @memory = (memory.keys | object.keys).map do |job|
+            [job,
+             memory[job]['average'],
+             object[job]['average'],
+             memory[job]['sum'],
+             object[job]['sum'],
+            ]
+          end.sort_by {|x| -x[3] }
+
+          render(:erb, File.read(File.join(view_path, "memory.erb")))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/web_extensions/throughput_monitor.rb

@@ -0,0 +1,32 @@
+module SidekiqUtils
+  module WebExtensions
+    module ThroughputMonitor
+      def self.registered(app)
+        view_path = File.join(File.expand_path("..", __FILE__), "views")
+
+        app.get("/throughput") do
+          last_run_at = {}
+          Sidekiq.redis {|r| r.hgetall('sidekiq_last_run') }.each do |job, last_run|
+            last_run_at[job] = last_run
+          end
+          @throughput = SidekiqUtils::RedisMonitorStorage.
+              retrieve('sidekiq_elapsed', 'elapsed').map do |job, values|
+
+            if last_run_at[job]
+              last_run_time = Time.at(Integer(last_run_at[job])).
+                in_time_zone('US/Eastern').to_s(:long) + ' ET'
+            else
+              last_run_time = 'n/a'
+            end
+            [job,
+             values['average'],
+             values['count'],
+             last_run_time,
+            ]
+          end.sort_by {|x| -x[2] }
+          render(:erb, File.read(File.join(view_path, "throughput.erb")))
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq_utils.rb

@@ -0,0 +1,21 @@
+require 'sidekiq_utils/middleware/client/additional_serialization'
+require 'sidekiq_utils/middleware/client/deprioritize'
+require 'sidekiq_utils/middleware/client/job_counter'
+
+require 'sidekiq_utils/middleware/server/additional_serialization'
+require 'sidekiq_utils/middleware/server/find_optional'
+require 'sidekiq_utils/middleware/server/job_counter'
+require 'sidekiq_utils/middleware/server/memory_monitor'
+require 'sidekiq_utils/middleware/server/throughput_monitor'
+
+require 'sidekiq_utils/web_extensions/job_counter'
+require 'sidekiq_utils/web_extensions/memory_monitor'
+require 'sidekiq_utils/web_extensions/throughput_monitor'
+
+require 'sidekiq_utils/redis_monitor_storage'
+require 'sidekiq_utils/additional_serialization'
+require 'sidekiq_utils/deprioritize'
+require 'sidekiq_utils/enqueued_jobs_helper'
+require 'sidekiq_utils/find_optional'
+require 'sidekiq_utils/job_counter'
+require 'sidekiq_utils/latency_alert'

data/sidekiq_utils.gemspec

@@ -0,0 +1,84 @@
+# Generated by juwelier
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+# stub: sidekiq_utils 1.0.0 ruby lib
+
+Gem::Specification.new do |s|
+  s.name = "sidekiq_utils".freeze
+  s.version = "1.0.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
+  s.require_paths = ["lib".freeze]
+  s.authors = ["Magnus von Koeller".freeze]
+  s.date = "2017-12-08"
+  s.description = "Tools that make working with a major Sidekiq installation more fun.".freeze
+  s.email = "magnus@angel.co".freeze
+  s.extra_rdoc_files = [
+    "LICENSE",
+    "README.md"
+  ]
+  s.files = [
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE",
+    "README.md",
+    "Rakefile",
+    "VERSION",
+    "lib/sidekiq_utils.rb",
+    "lib/sidekiq_utils/additional_serialization.rb",
+    "lib/sidekiq_utils/deprioritize.rb",
+    "lib/sidekiq_utils/enqueued_jobs_helper.rb",
+    "lib/sidekiq_utils/find_optional.rb",
+    "lib/sidekiq_utils/job_counter.rb",
+    "lib/sidekiq_utils/latency_alert.rb",
+    "lib/sidekiq_utils/middleware/client/additional_serialization.rb",
+    "lib/sidekiq_utils/middleware/client/deprioritize.rb",
+    "lib/sidekiq_utils/middleware/client/job_counter.rb",
+    "lib/sidekiq_utils/middleware/server/additional_serialization.rb",
+    "lib/sidekiq_utils/middleware/server/find_optional.rb",
+    "lib/sidekiq_utils/middleware/server/job_counter.rb",
+    "lib/sidekiq_utils/middleware/server/memory_monitor.rb",
+    "lib/sidekiq_utils/middleware/server/throughput_monitor.rb",
+    "lib/sidekiq_utils/redis_monitor_storage.rb",
+    "lib/sidekiq_utils/views/job_counts.erb",
+    "lib/sidekiq_utils/views/memory.erb",
+    "lib/sidekiq_utils/views/throughput.erb",
+    "lib/sidekiq_utils/web_extensions/job_counter.rb",
+    "lib/sidekiq_utils/web_extensions/memory_monitor.rb",
+    "lib/sidekiq_utils/web_extensions/throughput_monitor.rb",
+    "sidekiq_utils.gemspec"
+  ]
+  s.homepage = "http://github.com/venturehacks/sidekiq_angels".freeze
+  s.licenses = ["MIT".freeze]
+  s.rubygems_version = "2.6.13".freeze
+  s.summary = "Tools that make working with a major Sidekiq installation more fun.".freeze
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<sidekiq>.freeze, [">= 4.0.0"])
+      s.add_development_dependency(%q<shoulda>.freeze, [">= 0"])
+      s.add_development_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+      s.add_development_dependency(%q<bundler>.freeze, ["~> 1.0"])
+      s.add_development_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+      s.add_development_dependency(%q<simplecov>.freeze, [">= 0"])
+    else
+      s.add_dependency(%q<sidekiq>.freeze, [">= 4.0.0"])
+      s.add_dependency(%q<shoulda>.freeze, [">= 0"])
+      s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+      s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
+      s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+      s.add_dependency(%q<simplecov>.freeze, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<sidekiq>.freeze, [">= 4.0.0"])
+    s.add_dependency(%q<shoulda>.freeze, [">= 0"])
+    s.add_dependency(%q<rdoc>.freeze, ["~> 3.12"])
+    s.add_dependency(%q<bundler>.freeze, ["~> 1.0"])
+    s.add_dependency(%q<juwelier>.freeze, ["~> 2.1.0"])
+    s.add_dependency(%q<simplecov>.freeze, [">= 0"])
+  end
+end
+