sidekiq_cleaner 5.3.6

data/README.md

@@ -0,0 +1,55 @@
+(ReachLocal) Sidekiq Cleaner
+==============
+
+For additional documentation, see the [Sidekiq repository](https://github.com/mperham/sidekiq)
+
+Motivation
+-----------------
+Sidekiq is a very performant library for handling background processing of jobs, but could use with an improved UI for managing job statuses. The Capture team, found ourselves utilizing the [resque-cleaner]() gem frequently to manage our job processing, and felt that this should be included in the Sidekiq library. What is included here is a fork of the Sidekiq library with some tweaks to the `Web` rails engine under a new `Cleaner` namespace.
+
+Installation
+-----------------
+
+    gem install sidekiq-cleaner
+
+Usage
+-----------------
+Mount the Cleaner engine in the `config/routes.rb` file like so:
+
+```ruby
+Rails.application.routes.draw do
+  mount Sidekiq::Cleaner => '/cleaner'
+end
+```
+
+The web UI should appear familiar to users of the Sidekiq job library - the only difference is that a new Cleaner tab is introduced:
+
+![cleaner.png](./cleaner.png)
+
+This tab introduces a useful UI for managing job failures (as filtered by class and exception) en masse - which was only supported for one-off retries and deletes in the vanilla library. Perhaps a future release of Sidekiq will improve on this functionality!
+
+![cleaner-stats.png](./cleaner-stats.png)
+
+Useful resources:
+
+* Product documentation is in the [wiki](https://github.com/mperham/sidekiq/wiki).
+* Release announcements are made to the [@sidekiq](https://twitter.com/sidekiq) Twitter account.
+* The [Sidekiq tag](https://stackoverflow.com/questions/tagged/sidekiq) on Stack Overflow has lots of useful Q & A.
+
+Thanks
+-----------------
+
+Sidekiq stays fast by using the [JProfiler java profiler](http://www.ej-technologies.com/products/jprofiler/overview.html) to find and fix
+performance problems on JRuby.  Unfortunately MRI does not have good multithreaded profiling tools.
+
+
+License
+-----------------
+
+MIT
+
+
+Author
+-----------------
+
+Dan Belling, [danhbelling@gmail.com](mailto:danhbelling@gmail.com), [http://www.danbelling.me](http://danbelling.me)

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.warning = true
+  test.pattern = 'test/**/test_*.rb'
+end
+
+task default: :test

data/bin/sidekiq

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+
+require_relative '../lib/sidekiq/cli'
+
+begin
+  cli = Sidekiq::CLI.instance
+  cli.parse
+  cli.run
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end

data/bin/sidekiqctl

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'fileutils'
+require 'sidekiq/api'
+require 'sidekiq/ctl'
+
+if ARGV[0] == 'status'
+  Sidekiq::Ctl::Status.new.display(ARGV[1])
+else
+  if ARGV.length < 2
+    Sidekiq::Ctl.print_usage
+  else
+    stage = ARGV[0]
+    pidfile = ARGV[1]
+    timeout = ARGV[2].to_i
+    timeout = Sidekiq::Ctl::DEFAULT_KILL_TIMEOUT if timeout == 0
+
+    Sidekiq::Ctl.new(stage, pidfile, timeout)
+  end
+end

data/bin/sidekiqload

@@ -0,0 +1,149 @@
+#!/usr/bin/env ruby
+
+# Quiet some warnings we see when running in warning mode:
+# RUBYOPT=-w bundle exec sidekiq
+$TESTING = false
+
+#require 'ruby-prof'
+Bundler.require(:default)
+
+require_relative '../lib/sidekiq/cli'
+require_relative '../lib/sidekiq/launcher'
+
+include Sidekiq::Util
+
+Sidekiq.configure_server do |config|
+  #config.options[:concurrency] = 1
+  config.redis = { db: 13 }
+  config.options[:queues] << 'default'
+  config.logger.level = Logger::ERROR
+  config.average_scheduled_poll_interval = 2
+  config.reliable! if defined?(Sidekiq::Pro)
+end
+
+class LoadWorker
+  include Sidekiq::Worker
+  sidekiq_options retry: 1
+  sidekiq_retry_in do |x|
+    1
+  end
+
+  def perform(idx)
+    #raise idx.to_s if idx % 100 == 1
+  end
+end
+
+# brew tap shopify/shopify
+# brew install toxiproxy
+# gem install toxiproxy
+#require 'toxiproxy'
+# simulate a non-localhost network for realer-world conditions.
+# adding 1ms of network latency has an ENORMOUS impact on benchmarks
+#Toxiproxy.populate([{
+    #"name": "redis",
+    #"listen": "127.0.0.1:6380",
+    #"upstream": "127.0.0.1:6379"
+#}])
+
+self_read, self_write = IO.pipe
+%w(INT TERM TSTP TTIN).each do |sig|
+  begin
+    trap sig do
+      self_write.puts(sig)
+    end
+  rescue ArgumentError
+    puts "Signal #{sig} not supported"
+  end
+end
+
+Sidekiq.redis {|c| c.flushdb}
+def handle_signal(launcher, sig)
+  Sidekiq.logger.debug "Got #{sig} signal"
+  case sig
+  when 'INT'
+    # Handle Ctrl-C in JRuby like MRI
+    # http://jira.codehaus.org/browse/JRUBY-4637
+    raise Interrupt
+  when 'TERM'
+    # Heroku sends TERM and then waits 30 seconds for process to exit.
+    raise Interrupt
+  when 'TSTP'
+    Sidekiq.logger.info "Received TSTP, no longer accepting new work"
+    launcher.quiet
+  when 'TTIN'
+    Thread.list.each do |thread|
+      Sidekiq.logger.warn "Thread TID-#{(thread.object_id ^ ::Process.pid).to_s(36)} #{thread['label']}"
+      if thread.backtrace
+        Sidekiq.logger.warn thread.backtrace.join("\n")
+      else
+        Sidekiq.logger.warn "<no backtrace available>"
+      end
+    end
+  end
+end
+
+def Process.rss
+  `ps -o rss= -p #{Process.pid}`.chomp.to_i
+end
+
+iter = 10
+count = 10_000
+
+iter.times do
+  arr = Array.new(count) do
+    []
+  end
+  count.times do |idx|
+    arr[idx][0] = idx
+  end
+  Sidekiq::Client.push_bulk('class' => LoadWorker, 'args' => arr)
+end
+Sidekiq.logger.error "Created #{count*iter} jobs"
+
+Monitoring = Thread.new do
+  watchdog("monitor thread") do
+    while true
+      sleep 1
+      qsize, retries = Sidekiq.redis do |conn|
+        conn.pipelined do
+          conn.llen "queue:default"
+          conn.zcard "retry"
+        end
+      end.map(&:to_i)
+      total = qsize + retries
+      #GC.start
+      Sidekiq.logger.error("RSS: #{Process.rss} Pending: #{total}")
+      if total == 0
+        Sidekiq.logger.error("Done")
+        exit(0)
+      end
+    end
+  end
+end
+
+begin
+  #RubyProf::exclude_threads = [ Monitoring ]
+  #RubyProf.start
+  fire_event(:startup)
+  #Sidekiq.logger.error "Simulating 1ms of latency between Sidekiq and redis"
+  #Toxiproxy[:redis].downstream(:latency, latency: 1).apply do
+    launcher = Sidekiq::Launcher.new(Sidekiq.options)
+    launcher.run
+
+    while readable_io = IO.select([self_read])
+      signal = readable_io.first[0].gets.strip
+      handle_signal(launcher, signal)
+    end
+  #end
+rescue SystemExit => e
+  #Sidekiq.logger.error("Profiling...")
+  #result = RubyProf.stop
+  #printer = RubyProf::GraphHtmlPrinter.new(result)
+  #printer.print(File.new("output.html", "w"), :min_percent => 1)
+  # normal
+rescue => e
+  raise e if $DEBUG
+  STDERR.puts e.message
+  STDERR.puts e.backtrace.join("\n")
+  exit 1
+end