sidekiq-killswitch 1.0.0.pre1

data/sidekiq-killswitch.gemspec

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+require File.expand_path('../lib/sidekiq/killswitch/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = 'sidekiq-killswitch'
+  gem.version       = Sidekiq::Killswitch::VERSION
+  gem.summary       = 'Cross-host Sidekiq worker killswitches'
+  gem.authors       = ['Yuriy Naidyon']
+  gem.email         = 'yurokle@gmail.com'
+  gem.license       = 'Apache-2.0'
+  gem.homepage      = 'https://github.com/square/sidekiq-killswitch'
+
+  gem.files         = `git ls-files`.split($RS)
+  gem.test_files    = gem.files.grep(%r{spec/})
+  gem.require_paths = ['lib']
+  gem.required_ruby_version = '>= 2.2.2'
+  gem.metadata['allowed_push_host'] = 'https://rubygems.org'
+
+  gem.add_runtime_dependency 'sidekiq', '>= 3'
+
+  gem.add_development_dependency 'rspec', '~> 3.6'
+  gem.add_development_dependency 'rack-test'
+  gem.add_development_dependency 'rspec-html-matchers'
+  gem.add_development_dependency 'rubocop', '~> 0.49.1'
+end

data/spec/config_spec.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Sidekiq::Killswitch::Config do
+  let(:config) { Sidekiq::Killswitch::Config.new }
+
+  describe '#web_ui_worker_validator' do
+    it 'should to string presence validation by default' do
+      expect(config.web_ui_worker_validator.call(nil)).to be_falsey
+      expect(config.web_ui_worker_validator.call('')).to be_falsey
+      expect(config.web_ui_worker_validator.call('MyWorker')).to be_truthy
+    end
+  end
+
+  describe '#validate_worker_class_in_web' do
+    it 'should set Sidekiq::Worker module inclusion check as validator' do
+      stub_const('GoodWorker', Class.new {
+        include Sidekiq::Worker
+      })
+      stub_const('AlsoGoodWorker', Class.new(GoodWorker) {})
+      stub_const('BadWorker', Class.new {})
+
+      config.validate_worker_class_in_web
+
+      expect(config.web_ui_worker_validator.call('GoodWorker')).to be_truthy
+      expect(config.web_ui_worker_validator.call('AlsoGoodWorker')).to be_truthy
+
+      expect(config.web_ui_worker_validator.call('BadWorker')).to be_falsey
+      expect(config.web_ui_worker_validator.call('')).to be_falsey
+      expect(config.web_ui_worker_validator.call('Sidekiq::Worker')).to be_falsey
+    end
+  end
+end

data/spec/dead_set_spec.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Sidekiq::DeadSet do
+  let(:dead_set) { Sidekiq::DeadSet.new }
+
+  describe '#kill' do
+    it 'should put passed serialized job to the "dead" sorted set' do
+      serialized_job = Sidekiq.dump_json(jid: '123123', class: 'SomeWorker', args: [])
+      dead_set.kill(serialized_job)
+
+      expect(dead_set.find_job('123123').value).to eq(serialized_job)
+    end
+
+    it 'should remove dead jobs older than Sidekiq::DeadSet.timeout' do
+      allow(Sidekiq::DeadSet).to receive(:timeout).and_return(10)
+      time_now = Time.now
+
+      stub_time_now(time_now - 11)
+      dead_set.kill(Sidekiq.dump_json({jid: '000103', class: 'MyWorker3', args: []})) # the oldest
+
+      stub_time_now(time_now - 9)
+      dead_set.kill(Sidekiq.dump_json({jid: '000102', class: 'MyWorker2', args: []}))
+
+      stub_time_now(time_now)
+      dead_set.kill(Sidekiq.dump_json({jid: '000101', class: 'MyWorker1', args: []}))
+
+      stub_time_now(time_now)
+
+      expect(dead_set.find_job('000103')).to be_falsey
+      expect(dead_set.find_job('000102')).to be_truthy
+      expect(dead_set.find_job('000101')).to be_truthy
+    end
+
+    it 'should remove all but last Sidekiq::DeadSet.max_jobs-1 jobs' do
+      allow(Sidekiq::DeadSet).to receive(:max_jobs).and_return(3)
+
+      dead_set.kill(Sidekiq.dump_json({jid: '000101', class: 'MyWorker1', args: []}))
+      dead_set.kill(Sidekiq.dump_json({jid: '000102', class: 'MyWorker2', args: []}))
+      dead_set.kill(Sidekiq.dump_json({jid: '000103', class: 'MyWorker3', args: []}))
+
+      expect(dead_set.find_job('000101')).to be_falsey
+      expect(dead_set.find_job('000102')).to be_truthy
+      expect(dead_set.find_job('000103')).to be_truthy
+    end
+  end
+end

data/spec/killswitch_spec.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Sidekiq::Killswitch do
+  # !!!!! HERE BE DRAGONS !!!!!
+  # Make sure to return test config to the working state after testing ...
+  # ... consider using Rspec.around wrapper:
+  #
+  # around do |example|
+  #   original_test_logger = Sidekiq::Killswitch.logger
+  #   example.run
+  #   Sidekiq::Killswitch.logger = original_test_logger
+  # end
+
+  around do |example|
+    original_test_logger = Sidekiq::Killswitch.config.logger
+
+    example.run
+
+    Sidekiq::Killswitch.config.logger = original_test_logger
+  end
+
+  let(:worker_name) { 'SomeWorker' }
+
+  describe '.configure' do
+    describe '.logger= ' do
+      it 'should allow to set the logger' do
+        logger = double
+
+        Sidekiq::Killswitch.configure do |config|
+          config.logger = logger
+        end
+
+        expect(Sidekiq::Killswitch.logger).to eq(logger)
+      end
+    end
+  end
+
+  describe '.blackhole_add_worker' do
+    it 'should mark a worker as blackholed in Redis' do
+      time_now = stub_time_now
+
+      Sidekiq::Killswitch.blackhole_add_worker(worker_name)
+
+      Sidekiq::Killswitch.redis_pool do |redis|
+        expect(redis.hget(Sidekiq::Killswitch::BLACKHOLE_WORKERS_KEY_NAME, worker_name)).to eq(time_now.to_s)
+      end
+    end
+
+    it 'should accept class object as a parameter' do
+      time_now = stub_time_now
+      stub_const('AnotherWorker', Class.new {})
+
+      Sidekiq::Killswitch.blackhole_add_worker(AnotherWorker)
+
+      Sidekiq::Killswitch.redis_pool do |redis|
+        expect(redis.hget(Sidekiq::Killswitch::BLACKHOLE_WORKERS_KEY_NAME, 'AnotherWorker')).to eq(time_now.to_s)
+      end
+    end
+  end
+
+  describe '.blackhole_remove_worker' do
+    it 'should remove worker from the list of blackholed workers' do
+      Sidekiq::Killswitch.blackhole_add_worker(worker_name)
+      Sidekiq::Killswitch.blackhole_remove_worker(worker_name)
+
+      Sidekiq::Killswitch.redis_pool do |redis|
+        expect(redis.hexists(Sidekiq::Killswitch::BLACKHOLE_WORKERS_KEY_NAME, worker_name)).to be_falsey
+      end
+    end
+  end
+
+  describe '.blackhole_worker?' do
+    it 'should return true for blackholed workers' do
+      Sidekiq::Killswitch.blackhole_add_worker(worker_name)
+      expect(Sidekiq::Killswitch.blackhole_worker?(worker_name)).to be_truthy
+    end
+
+    it 'should return false for non-blackholed workers' do
+      expect(Sidekiq::Killswitch.blackhole_worker?(worker_name)).to be_falsey
+    end
+  end
+
+  describe '.blackhole_workers' do
+    it 'should return a list of all blackholed workers with "added at" timestamps' do
+      first_worker_added_at = stub_time_now
+      Sidekiq::Killswitch.blackhole_add_worker('FirstWorker')
+      second_worker_added_at = stub_time_now(first_worker_added_at + 1)
+      Sidekiq::Killswitch.blackhole_add_worker('SecondWorker')
+
+      expect(Sidekiq::Killswitch.blackhole_workers).to eq({
+        'FirstWorker' => first_worker_added_at.to_s,
+        'SecondWorker' => second_worker_added_at.to_s
+      })
+    end
+  end
+
+  describe '.dead_queue_add_worker' do
+    it 'should mark a worker as a "dead queue worker" in Redis' do
+      time_now = stub_time_now
+
+      Sidekiq::Killswitch.dead_queue_add_worker(worker_name)
+
+      Sidekiq::Killswitch.redis_pool do |redis|
+        expect(redis.hget(Sidekiq::Killswitch::DEAD_QUEUE_WORKERS_KEY_NAME, worker_name)).to eq(time_now.to_s)
+      end
+    end
+  end
+
+  describe '.dead_queue_remove_worker' do
+    it 'should remove worker from the list of dead queue workers' do
+      Sidekiq::Killswitch.dead_queue_add_worker(worker_name)
+      Sidekiq::Killswitch.dead_queue_remove_worker(worker_name)
+
+      Sidekiq::Killswitch.redis_pool do |redis|
+        expect(redis.hexists(Sidekiq::Killswitch::DEAD_QUEUE_WORKERS_KEY_NAME, worker_name)).to be_falsey
+      end
+    end
+  end
+
+  describe '.dead_queue_worker?' do
+    it 'should return true for dead queue workers' do
+      Sidekiq::Killswitch.dead_queue_add_worker(worker_name)
+      expect(Sidekiq::Killswitch.dead_queue_worker?(worker_name)).to be_truthy
+    end
+
+    it 'should return false for non-dead-queue workers' do
+      expect(Sidekiq::Killswitch.dead_queue_worker?(worker_name)).to be_falsey
+    end
+  end
+
+  describe '.dead_queue_workers' do
+    it 'should return a list of all dead queue workers with "added at" timestamps' do
+      first_worker_added_at = stub_time_now
+      Sidekiq::Killswitch.dead_queue_add_worker('FirstWorker')
+      second_worker_added_at = stub_time_now(first_worker_added_at + 1)
+      Sidekiq::Killswitch.dead_queue_add_worker('SecondWorker')
+
+      expect(Sidekiq::Killswitch.dead_queue_workers).to eq({
+        'FirstWorker' => first_worker_added_at.to_s,
+        'SecondWorker' => second_worker_added_at.to_s
+      })
+    end
+  end
+end

data/spec/middleware/client_spec.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Sidekiq::Killswitch::Middleware::Client do
+  let(:client_middleware) { Sidekiq::Killswitch::Middleware::Client.new }
+
+  describe '#call' do
+    it 'should return false for blackholed workers' do
+      stub_const('DisabledWorker', Class.new {})
+
+      Sidekiq::Killswitch.blackhole_add_worker(DisabledWorker)
+      Sidekiq::Killswitch.blackhole_add_worker('MyWorker')
+
+      expect(client_middleware.call(DisabledWorker, {}, nil, nil) { 123 }).to eq(false)
+      expect(client_middleware.call('MyWorker', {}, nil, nil) { 123 }).to eq(false)
+    end
+
+    it 'should return block result for not blockholed workers' do
+      stub_const('EnabledWorker', Class.new {})
+
+      expect(client_middleware.call(EnabledWorker, {}, nil, nil) { 'result' }).to eq('result')
+      expect(client_middleware.call('EnabledWorker', {}, nil, nil) { 123 }).to eq(123)
+    end
+  end
+end

data/spec/middleware/server_spec.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+require 'spec_helper'
+
+RSpec.describe Sidekiq::Killswitch::Middleware::Server do
+  let(:server_middleware) { Sidekiq::Killswitch::Middleware::Server.new }
+
+  before do
+    stub_const('MyWorker', Class.new {
+      include Sidekiq::Worker
+      def perform; end
+    })
+  end
+
+  describe '#call' do
+    context 'for blackholed workers' do
+      it 'should not run block' do
+        Sidekiq::Killswitch.blackhole_add_worker(MyWorker)
+
+        expect do
+          server_middleware.call(MyWorker.new, {}, nil) { raise 'Should not run' }
+        end.to_not raise_error
+      end
+    end
+
+    context 'for dead-queued workers' do
+      it 'should send a job to the morgue' do
+        Sidekiq::Killswitch.dead_queue_add_worker(MyWorker)
+
+        job_data = double
+        expect_any_instance_of(Sidekiq::DeadSet).to receive(:kill).with(job_data)
+
+        expect do
+          server_middleware.call(MyWorker.new, job_data, nil) { raise 'Should not run' }
+        end.to_not raise_error
+      end
+    end
+
+    context 'for non-marked workers' do
+      it 'should run passed block' do
+        checkpoint = double
+        expect(checkpoint).to receive(:check)
+        server_middleware.call(MyWorker.new, {}, nil) { checkpoint.check }
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+require 'rspec'
+require 'logger'
+require 'sidekiq/testing'
+require 'sidekiq/killswitch'
+
+ENV['RACK_ENV'] = 'test' # Disable CSRF protection for Sidekiq Web app
+
+Sidekiq::Killswitch.configure do |config|
+  config.logger = Logger.new('/dev/null')
+end
+
+RSpec.configure do |config|
+  config.disable_monkey_patching!
+  config.before do
+    Sidekiq::Killswitch.redis_pool { |c| c.flushdb }
+  end
+end
+
+def stub_time_now(time_now = Time.now)
+  allow(Time).to receive(:now).and_return(time_now)
+  time_now
+end

data/spec/web_spec.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+require 'spec_helper'
+require 'rack/test'
+require 'rspec-html-matchers'
+require 'sidekiq/killswitch/web'
+
+RSpec.describe Sidekiq::Killswitch::Web do
+  include Rack::Test::Methods
+  include RSpecHtmlMatchers
+
+  let(:app) { Sidekiq::Web }
+
+  def expect_redirect_to_root_page(response)
+    expect(response.status).to be(302)
+    expect(response.headers['Location']).to eq("http://#{rack_mock_session.default_host}/kill-switches")
+  end
+
+  describe 'GET /kill-switches' do
+    it 'should list blackholed workers' do
+      Sidekiq::Killswitch.blackhole_add_worker('WorkerOne')
+      Sidekiq::Killswitch.blackhole_add_worker('WorkerTwo')
+
+      response = get('/kill-switches')
+      expect(response.status).to be(200)
+
+      expect(response.body).to have_tag('.blackhole-workers') do
+        with_text('WorkerOne')
+        with_text('WorkerTwo')
+      end
+    end
+
+    it 'should list dead-queued workers' do
+      Sidekiq::Killswitch.dead_queue_add_worker('DeadWorkerOne')
+      Sidekiq::Killswitch.dead_queue_add_worker('DeadWorkerTwo')
+
+      response = get('/kill-switches')
+
+      expect(response.body).to have_tag('.dead-queue-workers') do
+        with_text('DeadWorkerOne')
+        with_text('DeadWorkerTwo')
+      end
+    end
+  end
+
+  describe 'POST /kill-switches/blackhole_add' do
+    it 'should blackhole passed worker' do
+      response = post('/kill-switches/blackhole_add', worker_name: 'BlackholedWorker')
+
+      expect_redirect_to_root_page(response)
+      expect(Sidekiq::Killswitch.blackhole_worker?('BlackholedWorker')).to be_truthy
+    end
+
+    describe 'validation' do
+      around do |example|
+        default_validator = Sidekiq::Killswitch.config.web_ui_worker_validator
+        example.run
+        Sidekiq::Killswitch.config.web_ui_worker_validator = default_validator
+      end
+
+      it 'should perform basic default validation' do
+        post('/kill-switches/blackhole_add', worker_name: '')
+        follow_redirect!
+
+        expect(last_response.body).to have_tag('.error-message', text: 'Error: Invalid worker name!')
+      end
+
+      it 'should perform custom validation' do
+        Sidekiq::Killswitch.config.web_ui_worker_validator = ->(name) { name.end_with?('BlockedWorker') }
+
+        post('/kill-switches/blackhole_add', worker_name: 'BadWorker')
+        follow_redirect!
+        expect(last_response.body).to have_tag('.error-message', text: 'Error: Invalid worker name!')
+
+        post('/kill-switches/blackhole_add', worker_name: 'MyBlockedWorker')
+        expect(Sidekiq::Killswitch.blackhole_worker?('MyBlockedWorker')).to be_truthy
+      end
+    end
+  end
+
+  describe 'POST /kill-switches/blackhole_remove' do
+    it 'should remove worker from blackhole' do
+      Sidekiq::Killswitch.blackhole_add_worker('BlakcholedWorker')
+
+      response = post('/kill-switches/blackhole_remove', worker_name: 'BlackholedWorker')
+
+      expect_redirect_to_root_page(response)
+      expect(Sidekiq::Killswitch.blackhole_worker?('BlackholedWorker')).to be_falsey
+    end
+  end
+
+  describe 'POST /kill-switches/dead_queue_add' do
+    it 'should add passed worker to dead queue' do
+      response = post('/kill-switches/dead_queue_add', worker_name: 'DeadWorker')
+
+      expect_redirect_to_root_page(response)
+      expect(Sidekiq::Killswitch.dead_queue_worker?('DeadWorker')).to be_truthy
+    end
+  end
+
+  describe 'POST /kill-switches/dead_queue_remove' do
+    it 'should remove worker from dead queue' do
+      Sidekiq::Killswitch.dead_queue_add_worker('DeadWorker')
+
+      response = post('/kill-switches/dead_queue_remove', worker_name: 'DeadWorker')
+
+      expect_redirect_to_root_page(response)
+      expect(Sidekiq::Killswitch.dead_queue_worker?('DeadWorker')).to be_falsey
+    end
+  end
+end

data/web/locales/en.yml

@@ -0,0 +1,15 @@
+en:
+  kill_switches: Kill Switches
+  blackholed_workers: Blackholed Workers
+  dead_queue_workers: Dead Queue Workers
+  are_you_sure: Are you sure?
+  blackhole: Blackhole
+  disabled_at: Disabled at
+  dead_queued_at: Dead-queued at
+  blackholed_at: Blackholed at
+  send_to_dead_queue: Send to dead queue
+  class: Class
+  kill: Kill
+  resurrect: Resurrect
+  resurrect_worker: Resurrect %{worker}?
+  invalid_worker_error: "Error: Invalid worker name!"

data/web/views/kill_switches.html.erb

@@ -0,0 +1,75 @@
+<% if @worker_name_invalid %>
+  <h4 class="error-message alert alert-danger"><%= t('invalid_worker_error')%></h4>
+<% end %>
+
+<h3><%= t('blackholed_workers')%></h3>
+
+<form action="<%= root_path %>kill-switches/blackhole_add" method="post" style="margin-bottom: 4px;">
+  <%= csrf_tag %>
+  <input name="worker_name" />
+  <input class="btn btn-danger btn-xs" type="submit" value="<%= t('blackhole')%>" data-confirm="<%= t('are_you_sure')%>" />
+</form>
+
+<div class="table_container">
+  <table class="blackhole-workers table table-hover table-bordered table-striped table-white">
+    <thead>
+      <tr>
+        <th><%= t('class')%></th>
+        <th style="width: 100%;"><%= t('blackholed_at')%></th>
+        <th></th>
+      </tr>
+    </thead>
+
+    <tbody>
+      <% @blackhole_workers.each do |worker_name, disabled_at| %>
+        <tr>
+          <td><%= worker_name %></td>
+          <td><%= disabled_at %></td>
+          <td class="text-center">
+            <form action="<%= root_path %>kill-switches/blackhole_remove" method="post">
+              <%= csrf_tag %>
+              <input type="hidden" name="worker_name" value="<%= worker_name %>" />
+              <input class="btn btn-primary btn-xs pull-right" type="submit" value="<%= t('restore')%>" data-confirm="<%= t('restore_worker', worker: worker_name) %>" />
+            </form>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>
+
+<h3><%= t('dead_queue_workers') %></h3>
+
+<form action="<%= root_path %>kill-switches/dead_queue_add" method="post" style="margin-bottom: 4px;">
+  <%= csrf_tag %>
+  <input name="worker_name" />
+  <input class="btn btn-danger btn-xs" type="submit" value="<%= t('send_to_dead_queue') %>" data-confirm="<%= t('are_you_sure')%>" />
+</form>
+
+<div class="table_container">
+  <table class="dead-queue-workers table table-hover table-bordered table-striped table-white">
+    <thead>
+    <tr>
+      <th><%= t('class')%></th>
+      <th style="width: 100%;"><%= t('dead_queued_at')%></th>
+      <th></th>
+    </tr>
+    </thead>
+
+    <tbody>
+      <% @dead_queue_workers.each do |worker_name, disabled_at| %>
+        <tr>
+          <td><%= worker_name %></td>
+          <td><%= disabled_at %></td>
+          <td class="text-center">
+            <form action="<%= root_path %>kill-switches/dead_queue_remove" method="post">
+              <%= csrf_tag %>
+              <input type="hidden" name="worker_name" value="<%= worker_name %>" />
+              <input class="btn btn-primary btn-xs pull-right" type="submit" value="<%= t('resurrect')%>" data-confirm="<%= t('resurrect_worker', worker: worker_name) %>" />
+            </form>
+          </td>
+        </tr>
+      <% end %>
+    </tbody>
+  </table>
+</div>