sidekiq_utils 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b3f6eac2a6e7d56402251588cfe585dd2ab0b489
+  data.tar.gz: 2b5c6aab41b220464b6ce061f66128be1a9ac006
+SHA512:
+  metadata.gz: 33f07d894defb62273a7722d19c4ee604cef8a4c9f9f17291a82e89f7614632f108fc58e8fdb8420a5407d6a0f5c07d18125e69b4a92f762614d24b9e6be52e7
+  data.tar.gz: 124dde56aa5b9bd3525974e42177b6a1ccd4ec4340d98fdee0fba8a4d0ae106825a560a5bdd0a111842297fbfdc1870ff6f5ad1dad5f439d631bc07ec68a9987

data/Gemfile

@@ -0,0 +1,16 @@
+source "https://rubygems.org"
+#
+# Add dependencies required to use your gem here.
+gem "sidekiq", ">= 4.0.0"
+# Example:
+#   gem "activesupport", ">= 2.3.5"
+
+# Add dependencies to develop your gem here.
+# Include everything needed to run rake, tests, features, etc.
+group :development do
+  gem "shoulda", ">= 0"
+  gem "rdoc", "~> 3.12"
+  gem "bundler", "~> 1.0"
+  gem "juwelier", "~> 2.1.0"
+  gem "simplecov", ">= 0"
+end

data/Gemfile.lock

@@ -0,0 +1,96 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (5.1.4)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (~> 0.7)
+      minitest (~> 5.1)
+      tzinfo (~> 1.1)
+    addressable (2.5.2)
+      public_suffix (>= 2.0.2, < 4.0)
+    builder (3.2.3)
+    concurrent-ruby (1.0.5)
+    connection_pool (2.2.1)
+    descendants_tracker (0.0.4)
+      thread_safe (~> 0.3, >= 0.3.1)
+    docile (1.1.5)
+    faraday (0.12.2)
+      multipart-post (>= 1.2, < 3)
+    git (1.3.0)
+    github_api (0.18.2)
+      addressable (~> 2.4)
+      descendants_tracker (~> 0.0.4)
+      faraday (~> 0.8)
+      hashie (~> 3.5, >= 3.5.2)
+      oauth2 (~> 1.0)
+    hashie (3.5.6)
+    highline (1.7.10)
+    i18n (0.9.1)
+      concurrent-ruby (~> 1.0)
+    json (1.8.6)
+    juwelier (2.1.3)
+      builder
+      bundler (>= 1.13)
+      git (>= 1.2.5)
+      github_api
+      highline (>= 1.6.15)
+      nokogiri (>= 1.5.10)
+      rake
+      rdoc
+      semver
+    jwt (1.5.6)
+    mini_portile2 (2.3.0)
+    minitest (5.10.3)
+    multi_json (1.12.2)
+    multi_xml (0.6.0)
+    multipart-post (2.0.0)
+    nokogiri (1.8.1)
+      mini_portile2 (~> 2.3.0)
+    oauth2 (1.4.0)
+      faraday (>= 0.8, < 0.13)
+      jwt (~> 1.0)
+      multi_json (~> 1.3)
+      multi_xml (~> 0.5)
+      rack (>= 1.2, < 3)
+    public_suffix (3.0.1)
+    rack (2.0.3)
+    rack-protection (2.0.0)
+      rack
+    rake (12.3.0)
+    rdoc (3.12.2)
+      json (~> 1.4)
+    redis (4.0.1)
+    semver (1.0.1)
+    shoulda (3.5.0)
+      shoulda-context (~> 1.0, >= 1.0.1)
+      shoulda-matchers (>= 1.4.1, < 3.0)
+    shoulda-context (1.2.2)
+    shoulda-matchers (2.8.0)
+      activesupport (>= 3.0.0)
+    sidekiq (5.0.5)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      rack-protection (>= 1.5.0)
+      redis (>= 3.3.4, < 5)
+    simplecov (0.15.1)
+      docile (~> 1.1.0)
+      json (>= 1.8, < 3)
+      simplecov-html (~> 0.10.0)
+    simplecov-html (0.10.2)
+    thread_safe (0.3.6)
+    tzinfo (1.2.4)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (~> 1.0)
+  juwelier (~> 2.1.0)
+  rdoc (~> 3.12)
+  shoulda
+  sidekiq (>= 4.0.0)
+  simplecov
+
+BUNDLED WITH
+   1.16.0

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 VentureHacks
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,185 @@
+# sidekiq_utils
+Sidekiq powers our background processing needs at AngelList. As we introduced Sidekiq in a legacy codebase and it started to handle significant job throughput, we developed a number of utilities to make working with Sidekiq easier that we'd love to share.
+
+## Additional Serialization
+
+Adds support for automatically serializing and deserializing additional argument types for your workers:
+* `Class`
+* `Symbol` (both by themselves and as hash keys)
+* `ActiveSupport::HashWithIndifferentAccess`
+
+To use this utility, you need to include the client and server middlewares. Be sure to include the client middleware on the server as well.
+
+### Configuration
+```
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Server::AdditionalSerialization
+  end
+  config.client_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Client::AdditionalSerialization
+  end
+end
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Client::AdditionalSerialization
+  end
+end
+```
+
+### A note of caution for users of the sidekiq-unique-jobs gem
+
+sidekiq-unique-jobs will fail to properly determine job uniqueness if you don't hook the middlewares in the correct order. If you use both sidekiq-unique-jobs and the additional serialization utility, be sure to hook your middlewares in the following way:
+```
+Sidekiq.configure_server do |config|
+  # unwrap arguments first
+  config.server_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Server::AdditionalSerialization
+  end
+  # then determine unique jobs
+  SidekiqUniqueJobs.configure_server_middleware
+
+  # first determine uniqueness
+  SidekiqUniqueJobs.configure_client_middleware
+  # then wrap arguments
+  config.client_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Client::AdditionalSerialization
+  end
+end
+Sidekiq.configure_client do |config|
+  # first determine uniqueness
+  SidekiqUniqueJobs.configure_client_middleware
+  # then wrap arguments
+  config.client_middleware do |chain|
+    chain.add ::SidekiqUtils::Middleware::Client::AdditionalSerialization
+  end
+end
+```
+
+## Deprioritize
+
+Adds an easy ability to divert jobs added within a block to a lower-priority queue. This is useful, for example, in cron jobs. All jobs added within the block will be added to the `low` queue instead of their default queue.
+
+### Configuration
+
+```
+Sidekiq.configure_server do |config|
+  config.client_middleware do |chain|
+    chain.add SidekiqUtils::Middleware::Client::Deprioritize
+  end
+end
+Sidekiq.configure_client do |config|
+  config.client_middleware do |chain|
+    client_middlewares.each do |middleware|
+      chain.add SidekiqUtils::Middleware::Client::Deprioritize
+    end
+  end
+end
+```
+
+### Usage
+
+```
+SidekiqUtils::Deprioritize.workers(SolrIndexWorker) do
+  10_000.times do
+    # these will all go to the `low` queue
+    SolrIndexWorker.perform_async(User, rand(10_000))
+  end
+end
+```
+
+## Enqueued jobs helper
+
+A simple tool to inspect and manipulate Sidekiq queues from the console:
+
+```
+> SidekiqUtils::EnqueuedJobsHelper.counts
+=> {"default"=>{}, "high"=>{"AlgoliaIndexWorker[JobProfile]"=>1}, "low"=>{"AlgoliaIndexWorker[JobProfile]"=>1}}
+
+> SidekiqUtils::EnqueuedJobsHelper.delete(queue: 'low', job_class: 'AlgoliaIndexWorker', first_argument: JobProfile)
+# first_argument is optional
+```
+
+## Find optional
+
+Lots of jobs become moot when a record cannot be found; however, because Sidekiq is very fast and it's easy to forget to enqueue jobs in `after_commit` hooks, sometimes the record isn't found simply because the transaction in which it was inserted has not been committed yet. This will retry a `find_optional` call exactly once after 30 seconds if the record cannot be found.
+
+### Configuration
+
+```
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add SidekiqUtils::Middleware::Server::FindOptional
+  end
+end
+```
+
+### Usage
+
+```
+class SolrIndexWorker
+  include Sidekiq::Worker
+  
+  def perform(user_id)
+    user = find_optional(User, user_id)
+  end
+end
+```
+
+## Job counter
+
+This will keep a running count of jobs per queue and worker class so that you can inspect what is clogging up high-volume queues without having to iterate over the entire queue.
+
+### Configuration
+
+```
+SidekiqUtils::JobCounter.hook_sidekiq!
+Sidekiq::Web.register SidekiqUtils::WebExtensions::JobCounter
+Sidekiq::Web.tabs["Job counts"] = "job_counts"
+```
+
+### Usage
+
+This will add a "Job counts" tab to your Sidekiq admin which will display the current job count.
+
+Please note that the accuracy of these numbers is not guaranteed as it is non-trivial to keep a running count. However, this is a useful tool when you are inspecting a very long queue.
+
+## Memory monitor
+
+This automatically checks memory usage before and after a worker is run and keeps track of which jobs consistently leak memory. Please note that this is very approximate. It also requires you running one worker process single-threaded with `-c 1`. It slows down jobs processed by that worker considerably.
+
+### Configuration
+
+```
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add SidekiqUtils::Middleware::Server::MemoryMonitor
+  end
+end
+Sidekiq::Web.register SidekiqUtils::WebExtensions::MemoryMonitor
+Sidekiq::Web.tabs["Memory"] = "memory"
+```
+
+### Usage
+
+This will add a "Memory" tab to your Sidekiq admin which will display memory usage information.
+
+## Throughput monitor
+
+This will keep track of how many jobs of which worker class have run in the past week, as well as when it was last run.
+
+### Configuration
+
+```
+Sidekiq.configure_server do |config|
+  config.server_middleware do |chain|
+    chain.add SidekiqUtils::Middleware::Server::ThroughputMonitor
+  end
+end
+Sidekiq::Web.register SidekiqUtils::WebExtensions::ThroughputMonitor
+Sidekiq::Web.tabs["Throughput"] = "throughput"
+```
+
+### Usage
+
+This will add a "Throughput" tab to your Sidekiq admin which will display job throughput information.

data/Rakefile

@@ -0,0 +1,50 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+require 'juwelier'
+Juwelier::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://guides.rubygems.org/specification-reference/ for more options
+  gem.name = "sidekiq_utils"
+  gem.homepage = "http://github.com/venturehacks/sidekiq_angels"
+  gem.license = "MIT"
+  gem.summary = %Q{Tools that make working with a major Sidekiq installation more fun.}
+  gem.description = %Q{Tools that make working with a major Sidekiq installation more fun.}
+  gem.email = "magnus@angel.co"
+  gem.authors = ["Magnus von Koeller"]
+
+  # dependencies defined in Gemfile
+end
+Juwelier::RubygemsDotOrgTasks.new
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+desc "Code coverage detail"
+task :simplecov do
+  ENV['COVERAGE'] = "true"
+  Rake::Task['test'].execute
+end
+
+task :default => :test
+
+require 'rdoc/task'
+Rake::RDocTask.new do |rdoc|
+  version = File.exist?('VERSION') ? File.read('VERSION') : ""
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "sidekiq_angels #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.0.0

data/lib/sidekiq_utils/additional_serialization.rb

@@ -0,0 +1,56 @@
+module SidekiqUtils
+  module AdditionalSerialization
+    def self.wrap_argument(arg)
+      if arg.is_a?(Array)
+        arg.map {|a| wrap_argument(a) }
+      elsif arg.is_a?(Hash)
+        wrapped = arg.each_with_object({}) do |(key, value), hash|
+          hash[key.to_s] = wrap_argument(value)
+        end
+        symbol_keys = arg.each_key.grep(Symbol).map(&:to_s)
+        wrapped['_al_aj_symbol_keys'] = symbol_keys if symbol_keys.present?
+        if arg.is_a?(ActiveSupport::HashWithIndifferentAccess)
+          wrapped['_al_aj_indifferent_access'] = true
+        end
+        wrapped
+      elsif arg.is_a?(Symbol)
+        { '_al_aj_wrapped' => 'symbol', 'value' => arg.to_s }
+      elsif arg.is_a?(Class)
+        { '_al_aj_wrapped' => 'class', 'value' => arg.name }
+      else
+        arg
+      end
+    end
+
+    def self.unwrap_argument(arg)
+      if arg.is_a?(Hash) && arg['_al_aj_wrapped'].present?
+        case arg['_al_aj_wrapped']
+        when 'symbol'
+          arg['value'].to_sym
+        when 'class'
+          arg['value'].constantize
+        else
+          fail("Unknown wrapped value: #{arg['_al_aj_wrapped']}")
+        end
+      elsif arg.is_a?(Hash)
+        # make sure that we don't accidentally mess with the argument here,
+        # rather the caller should be responsible for actually replacing values
+        arg = arg.deep_dup
+
+        symbol_keys = arg.delete('_al_aj_symbol_keys') || []
+        unwrapped = arg.each_with_object({}) do |(key, value), hash|
+          key = key.to_sym if symbol_keys.include?(key)
+          hash[key] = unwrap_argument(value)
+        end
+        if unwrapped.delete('_al_aj_indifferent_access')
+          unwrapped = unwrapped.with_indifferent_access
+        end
+        unwrapped
+      elsif arg.is_a?(Array)
+        arg.map {|a| unwrap_argument(a) }
+      else
+        arg
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/deprioritize.rb

@@ -0,0 +1,19 @@
+module SidekiqUtils
+  module Deprioritize
+    def self.workers(*workers)
+      workers = workers.map do |worker|
+        if worker.is_a?(Class)
+          worker.name
+        else
+          worker
+        end
+      end
+
+      old_deprioritized = Thread.current[:deprioritize_worker_classes]
+      Thread.current[:deprioritize_worker_classes] ||= []
+      Thread.current[:deprioritize_worker_classes] |= workers
+      yield
+      Thread.current[:deprioritize_worker_classes] = old_deprioritized
+    end
+  end
+end

data/lib/sidekiq_utils/enqueued_jobs_helper.rb

@@ -0,0 +1,45 @@
+module SidekiqUtils
+  module EnqueuedJobsHelper
+    class << self
+      def counts
+        counts = {}
+        Sidekiq::Queue.all.each do |queue|
+          queue_job_counts = counts[queue.name] ||= {}
+          queue.each do |job|
+            job_key = SidekiqUtils::RedisMonitorStorage.job_prefix(
+                job, unwrap_arguments: true)
+            queue_job_counts[job_key] ||= 0
+            queue_job_counts[job_key] += 1
+          end
+        end
+        counts
+      end
+
+      def delete(queue:, job_class:, first_argument: nil)
+        if job_class.is_a?(Class)
+          job_class = job_class.name
+        else
+          job_class = job_class.to_s
+        end
+
+        deleted = 0
+        Sidekiq::Queue.all.each do |iter_queue|
+          next if queue && queue.to_s != iter_queue.name
+
+          iter_queue.each do |job|
+            next if job['class'] != job_class
+            if first_argument && SidekiqUtils::AdditionalSerialization.
+                unwrap_argument(job['args'].first) != first_argument
+              next
+            end
+
+            job.delete
+            deleted += 1
+          end
+        end
+
+        deleted
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/find_optional.rb

@@ -0,0 +1,24 @@
+module SidekiqUtils
+  module FindOptional
+    extend ActiveSupport::Concern
+
+    class NotFoundError < StandardError; end
+
+    # try finding a record, but eventually give up retrying if we still cannot
+    # find it. use this if you are trying to load a record from a job, but
+    # don't want the failed job to end up in the RetrySet if it keeps failing.
+    def find_optional(entity, id, scope: nil)
+      entity = entity.public_send(scope) if scope
+      if id.is_a?(Enumerable)
+        instance = entity.where(id: id)
+      else
+        instance = entity.find_by(id: id)
+      end
+      if !instance.present?
+        fail(NotFoundError)
+      else
+        instance
+      end
+    end
+  end
+end

data/lib/sidekiq_utils/job_counter.rb

@@ -0,0 +1,161 @@
+module SidekiqUtils
+  module JobCounter
+    REDIS_KEY = 'sidekiq_utils_job_counter'
+    LOCK = Mutex.new
+    SYNC_COUNTS_EVERY = 1.second
+
+    class << self
+      def increment(job)
+        change_count(job, 1)
+      end
+
+      def decrement(job)
+        change_count(job, -1)
+      end
+
+      def counts
+        Sidekiq.redis do |redis|
+          redis.hgetall(REDIS_KEY).each_with_object({}) do |(key, count), ret_hash|
+            key_hash = JSON.parse(key, symbolize_names: true)
+
+            count = count.to_i
+            if count != 0
+              ret_hash[key_hash[:queue]] ||= {}
+              ret_hash[key_hash[:queue]][key_hash[:job]] = count
+            end
+          end
+        end
+      end
+
+      def reset!
+        Sidekiq.redis do |redis|
+          redis.del(REDIS_KEY)
+        end
+      end
+
+      def hook_sidekiq!
+        Sidekiq::SortedEntry.prepend(SortedEntry::JobCounterExtension)
+        Sidekiq::ScheduledSet.prepend(ScheduledSet::JobCounterExtension)
+        Sidekiq::Job.prepend(Job::JobCounterExtension)
+        Sidekiq::Queue.prepend(Queue::JobCounterExtension)
+        Sidekiq::JobSet.prepend(JobSet::JobCounterExtension)
+      end
+
+      private
+      def change_count(job, change_by)
+        unless [-1, 1].include?(change_by)
+          fail("Unsupported change_by value: #{change_by}")
+        end
+
+        job_key = SidekiqUtils::RedisMonitorStorage.job_prefix(
+            job, unwrap_arguments: true)
+        hash_key = {
+          queue: job['queue'],
+          job: job_key,
+        }.to_json
+
+        LOCK.synchronize do
+          @counts_to_flush ||= {}
+          @counts_to_flush[hash_key] ||= 0
+          @counts_to_flush[hash_key] += change_by
+
+          if !Rails.env.test?
+            @sync_thread ||= Thread.new do
+              sleep SYNC_COUNTS_EVERY
+              sync_to_redis
+            end
+          end
+        end
+
+        sync_to_redis if Rails.env.test?
+      end
+
+      def sync_to_redis
+        local_counts_to_flush = nil
+        LOCK.synchronize do
+          local_counts_to_flush = @counts_to_flush
+          @counts_to_flush = {}
+          @sync_thread = nil
+        end
+        (local_counts_to_flush || {}).each do |hash_key, change_by|
+          Sidekiq.redis do |redis|
+            count = redis.hincrby(REDIS_KEY, hash_key, change_by)
+            if count < 0 && change_by < 0
+              # this shouldn't happen, but it could when we first deploy this.
+              # just makes sure we don't end up with negative values here
+              # which don't make sense
+              #
+              # we only ever increment by the same amount we just did because
+              # we can't be responsible for more of a discrepancy at this
+              # point and we don't want multiple threads to overcorrect for
+              # each other
+              count = redis.hincrby(REDIS_KEY, hash_key, -1*change_by)
+            end
+          end
+        end
+      end
+      at_exit { SidekiqUtils::JobCounter.send(:sync_to_redis) }
+    end
+  end
+
+  class SortedEntry
+    module JobCounterExtension
+      def delete
+        if super
+          JobCounter.decrement(item)
+        end
+      end
+
+      private
+
+      def remove_job
+        super do |message|
+          JobCounter.decrement(Sidekiq.load_json(message))
+          yield message
+        end
+      end
+    end
+  end
+
+  class ScheduledSet
+    module JobCounterExtension
+      def delete
+        if super
+          JobCounter.decrement(item)
+        end
+      end
+    end
+  end
+
+  class Job
+    module JobCounterExtension
+      def delete
+        super
+        JobCounter.decrement(item)
+      end
+    end
+  end
+
+  class Queue
+    module JobCounterExtension
+      def clear
+        super
+      end
+    end
+  end
+
+  class JobSet
+    module JobCounterExtension
+      def clear
+        each(&:delete)
+        super
+      end
+
+      def delete_by_value(name, value)
+        if super
+          JobCounter.decrement(Sidekiq.load_json(value))
+        end
+      end
+    end
+  end
+end