sidekiq-undertaker 1.0.0.rc01

data/spec/sidekiq/undertaker/bucket_spec.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+module Sidekiq
+  module Undertaker
+    describe Bucket do
+      describe ".bucket_names" do
+        let(:expected_bucket_names) { %w[1_hour 3_hours 1_day 3_days 1_week older] }
+
+        it { expect(described_class.bucket_names).to eq expected_bucket_names }
+      end
+
+      describe ".for_elapsed_time" do
+        let(:expectation) { %w[1_hour 3_hours 1_day 3_days 1_week older] }
+        let(:elapsed_times) { [1, 3601, 10801, 86401, 259201, 604801] }
+
+        it "maps elapsed_time to bucket name" do
+          elapsed_times.each_with_index do |elapsed_time, index|
+            expect(described_class.for_elapsed_time(elapsed_time)).to eq expectation[index]
+          end
+        end
+      end
+    end
+  end
+end

data/spec/sidekiq/undertaker/dead_jobs_spec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+module Sidekiq
+  module Undertaker
+    describe DeadJob do
+      let(:job) do
+        build_job(
+          "class"       => "HardWorkTask",
+          "failed_at"   => Time.now,
+          "error_class" => "NoMethodError",
+          "queue"       => "SomeQueue"
+        )
+      end
+
+      describe "attributes" do
+        let(:expected_attributes) do
+          {
+            job_class:                  "HardWorkTask",
+            time_elapsed_since_failure: 9,
+            error_class:                "NoMethodError",
+            bucket_name:                "1_hour",
+            job:                        job
+          }
+        end
+
+        context "when all attributes are given" do
+          let(:args) { expected_attributes }
+
+          it { expect(described_class.new(args)).to have_attributes expected_attributes }
+        end
+
+        context "when some attributes are derived" do
+          let(:args) do
+            {
+              time_elapsed_since_failure: 9,
+              bucket_name:                "1_hour",
+              job:                        job
+            }
+          end
+
+          it do
+            expect(
+              described_class.new(args)
+            ).to have_attributes expected_attributes
+          end
+        end
+      end
+
+      describe ".for_each" do
+        let(:expected_dead_job) do
+          DeadJob.new(
+            job_class:                  "HardWorkTask",
+            time_elapsed_since_failure: time_elapsed,
+            error_class:                "NoMethodError",
+            bucket_name:                "1_hour",
+            job:                        killed_job
+          )
+        end
+
+        let(:killed_job) do
+          Sidekiq::DeadSet.new.find_job(job.jid)
+        end
+
+        let(:time_elapsed) { Time.now.to_i - job.item["failed_at"].to_i }
+
+        before do
+          Timecop.freeze
+
+          kill_job(job)
+        end
+
+        after do
+          Timecop.return
+        end
+
+        it "returns job with appropriate metadata filled" do
+          cnt = 0
+          described_class.for_each do |dead_job|
+            # It returns only one dead_job
+            cnt += 1
+            expect(dead_job).to eql expected_dead_job
+          end
+          expect(cnt).to be 1
+        end
+      end
+    end
+  end
+end

data/spec/sidekiq/undertaker/job_distributor_spec.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+module Sidekiq
+  module Undertaker
+    describe JobDistributor do
+      let(:job1) do
+        instance_double(Sidekiq::Job, item: {
+                          "class"       => "A",
+                          "failed_at"   => 1,
+                          "error_class" => "E1"
+                        })
+      end
+      let(:job2) do
+        instance_double(Sidekiq::Job, item: {
+                          "class"       => "A",
+                          "failed_at"   => 1,
+                          "error_class" => "E1"
+                        })
+      end
+      let(:job3) do
+        instance_double(Sidekiq::Job, item: {
+                          "class"       => "B",
+                          "failed_at"   => 1,
+                          "error_class" => "E1"
+                        })
+      end
+      let(:job4) do
+        instance_double(Sidekiq::Job, item: {
+                          "class"       => "B",
+                          "failed_at"   => 1,
+                          "error_class" => "E2"
+                        })
+      end
+
+      let(:dead_job1) do
+        DeadJob.new(
+          job:                        job1, # 'A', 'E1'
+          time_elapsed_since_failure: 10,
+          bucket_name:                "1_hour"
+        )
+      end
+
+      let(:dead_job2) do
+        DeadJob.new(
+          job:                        job2, # 'A', 'E1'
+          time_elapsed_since_failure: 10 + 60 * 60,
+          bucket_name:                "3_hours"
+        )
+      end
+      let(:dead_job3) do
+        DeadJob.new(
+          job:                        job3, # 'B', 'E1'
+          time_elapsed_since_failure: 10 + 60 * 60,
+          bucket_name:                "3_hours"
+        )
+      end
+      let(:dead_job4) do
+        DeadJob.new(
+          job:                        job4, # 'B', 'E2'
+          time_elapsed_since_failure: 10 + 60 * 60 * 24,
+          bucket_name:                "1_day"
+        )
+      end
+
+      let(:dead_jobs) { [dead_job1, dead_job2, dead_job3, dead_job4] }
+
+      # rubocop:disable Metrics/LineLength
+      describe "#group_by_job_class" do
+        subject(:distribution) { described_class.new(dead_jobs).group_by_job_class }
+
+        let(:expected_distribution) do
+          [
+            ["all", { "1_hour" => 1, "3_hours" => 2, "1_day" => 1, "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 4 }],
+            ["B",   { "1_hour" => 0, "3_hours" => 1, "1_day" => 1, "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 2 }],
+            ["A",   { "1_hour" => 1, "3_hours" => 1, "1_day" => 0, "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 2 }]
+          ]
+        end
+
+        it "distributes the dead jobs into buckets and groups them by job_class" do
+          expect(distribution).to eq expected_distribution
+        end
+      end
+
+      describe "#group_by_error_class" do
+        subject(:distribution) { described_class.new(dead_jobs).group_by_error_class }
+
+        let(:expected_distribution) do
+          [
+            ["all", { "1_hour" => 1, "3_hours" => 2, "1_day" => 1,  "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 4 }],
+            ["E1",  { "1_hour" => 1, "3_hours" => 2, "1_day" => 0,  "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 3 }],
+            ["E2",  { "1_hour" => 0, "3_hours" => 0, "1_day" => 1,  "3_days" => 0, "1_week" => 0, "older" => 0, "total_dead" => 1 }]
+          ]
+        end
+
+        it "distributes the dead jobs into buckets and groups them by error_class" do
+          expect(distribution).to eq expected_distribution
+        end
+      end
+      # rubocop:enable Metrics/LineLength
+    end
+  end
+end

data/spec/sidekiq/undertaker/job_filter_spec.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+module Sidekiq
+  module Undertaker
+    describe JobFilter do
+      describe ".filter_dead_jobs" do
+        let(:job1) do
+          instance_double(Sidekiq::Job, item: {
+                            "class"       => "HardWorkTask",
+                            "failed_at"   => Time.now.to_i - 5 * 60,
+                            "error_class" => "NoMethodError"
+                          })
+        end
+
+        let(:job2) do
+          instance_double(Sidekiq::Job, item: {
+                            "class"       => "HardWorkTask",
+                            "failed_at"   => Time.now.to_i - 2 * 60 * 60,
+                            "error_class" => "RandomError"
+                          })
+        end
+
+        let(:job3) do
+          instance_double(Sidekiq::Job, item: {
+                            "class"       => "LazyWorkTask",
+                            "failed_at"   => Time.now.to_i - 2 * 60 * 60,
+                            "error_class" => "NoMethodError"
+                          })
+        end
+
+        before do
+          Timecop.freeze
+
+          dead_set = instance_double(Sidekiq::DeadSet)
+
+          allow(dead_set).to receive(:each)
+            .and_yield(job1)
+            .and_yield(job2)
+            .and_yield(job3)
+
+          allow(Sidekiq::DeadSet).to receive(:new).and_return(dead_set)
+        end
+
+        after { Timecop.return }
+
+        context "when the job_class filter is given" do
+          subject(:dead_jobs) { described_class.filter_dead_jobs("job_class" => "HardWorkTask") }
+
+          it "filters jobs based on job_class" do
+            dead_jobs.each do |dead_job|
+              expect(dead_job.job_class).to eq "HardWorkTask"
+            end
+          end
+
+          it { expect(dead_jobs.size).to eq 2 }
+        end
+
+        context "when the job_class and bucket_name filters are given" do
+          subject(:dead_jobs) do
+            described_class.filter_dead_jobs("job_class"   => "HardWorkTask",
+                                             "bucket_name" => "3_hours")
+          end
+
+          it "filters based on multiple filter attributes" do
+            dead_jobs.each do |dead_job|
+              expect(dead_job.job_class).to eq "HardWorkTask"
+              expect(dead_job.bucket_name).to eq "3_hours"
+            end
+          end
+
+          it { expect(dead_jobs.size).to eq 1 }
+        end
+
+        context "when the error_class filter is given" do
+          subject(:dead_jobs) { described_class.filter_dead_jobs("error_class" => "NoMethodError") }
+
+          it "filters jobs based on error_class" do
+            dead_jobs.each do |dead_job|
+              expect(dead_job.error_class).to eq "NoMethodError"
+            end
+          end
+          it { expect(dead_jobs.size).to eq 2 }
+        end
+
+        context "when no filters are applied" do
+          subject(:dead_jobs) { described_class.filter_dead_jobs }
+
+          it { expect(dead_jobs.size).to eq 3 }
+        end
+
+        context "when all filters are nil" do
+          subject(:dead_jobs) do
+            described_class.filter_dead_jobs("job_class"   => nil,
+                                             "bucket_name" => nil,
+                                             "error_class" => nil)
+          end
+
+          it { expect(dead_jobs.size).to eq 3 }
+        end
+      end
+    end
+  end
+end

data/spec/sidekiq/undertaker/web_extension_spec.rb

@@ -0,0 +1,270 @@
+# frozen_string_literal: true
+
+require "spec_helper"
+
+require "sidekiq/api"
+require "sidekiq/web"
+require "sinatra"
+require "rack/test"
+
+module Sidekiq
+  # rubocop:disable Metrics/ModuleLength
+  module Undertaker
+    describe WebExtension, type: :controller do
+      include Rack::Test::Methods
+
+      let(:app) { Sidekiq::Web }
+      let(:job_refs) { [] }
+
+      let(:jid1) { "4416aa76eb8cf03f56a49220" }
+      let(:jid2) { "34e79a46b1956d3a1180767b" }
+      let(:jid3) { "8d08674fce759ac75d1a6e75" }
+      let(:jid4) { "bfa4a272cdcac8bfac7b9f1a" }
+
+      let(:default_job_opts) do
+        {
+          "class"         => "HardWorker",
+          "args"          => ["asdf", 1234],
+          "queue"         => "foo",
+          "error_message" => "Some fake message",
+          "error_class"   => "RuntimeError",
+          "retry_count"   => 0,
+          "failed_at"     => Time.now.utc
+        }
+      end
+
+      # rubocop:disable RSpec/AnyInstance
+      before do
+        Timecop.freeze(Time.gm(2018, 12, 16, 20, 57))
+
+        job_refs.push add_dead("jid" => jid1)
+        job_refs.push add_dead("jid" => jid2)
+        job_refs.push add_dead("jid" => jid3, "error_class" => "NoMethodError")
+        job_refs.push add_dead("jid" => jid4, "class" => "HardWorker1", "error_class" => "NoMethodError")
+
+        allow_any_instance_of(Sidekiq::WebAction).to receive(:root_path).and_return("/sidekiq/")
+      end
+      # rubocop:enable RSpec/AnyInstance
+
+      after { Timecop.return }
+
+      def add_dead(opts = {})
+        opts = default_job_opts.merge!(opts)
+
+        job = build_job(opts)
+        killed_job = kill_job(job)
+
+        "#{killed_job.score}-#{job.jid}"
+      end
+
+      shared_examples "a page" do
+        it "the displayed page is correct" do
+          subject
+
+          expect(last_response.status).to eq 200
+          verify do
+            apply_custom_excludes(last_response.body)
+          end
+        end
+      end
+
+      describe "show filter" do
+        # /undertaker/filter
+        context "when filter page is called" do
+          subject { get "/undertaker/filter" }
+
+          it_behaves_like "a page"
+        end
+
+        # /undertaker/filter/:job_class/:bucket_name
+        context "when job-class/bucket page is called" do
+          subject { get "/undertaker/filter/HardWorker/1_hour" }
+
+          it_behaves_like "a page"
+        end
+
+        # /undertaker/filter/:job_class/:bucket_name?poll=true
+        context "when job-class/bucket page is polled" do
+          subject { get "/undertaker/filter/HardWorker/1_hour?poll=true" }
+
+          it_behaves_like "a page"
+        end
+      end
+
+      describe "show morgue" do
+        # /undertaker/morgue/:job_class/:error_class/:bucket_name
+        context "when job-class/error/bucket is called" do
+          context "with specific job-class and a specific error" do
+            subject { get "/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+
+            it_behaves_like "a page"
+          end
+
+          context "with all failures and errors" do
+            subject { get "/undertaker/morgue/all/all/total_dead" }
+
+            it_behaves_like "a page"
+          end
+        end
+      end
+
+      describe "delete" do
+        context "when job-class, error and bucket are given" do
+          subject { post "/undertaker/morgue/HardWorker/RuntimeError/1_hour/delete" }
+
+          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+
+          let(:params) { { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour" } }
+          let(:dead_jobs_set) { [dead_job1, dead_job2] }
+          let(:dead_job1) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid1)) }
+          let(:dead_job2) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid2)) }
+
+          before do
+            allow(Sidekiq::Undertaker::JobFilter).to receive(:filter_dead_jobs).with(params)
+                                                                               .and_return(dead_jobs_set)
+          end
+
+          it "deletes the dead jobs" do
+            expect(dead_job1.job).to receive(:delete).and_call_original
+            expect(dead_job2.job).to receive(:delete).and_call_original
+            subject
+          end
+
+          it "reduces the DeadSet" do
+            expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(2)
+          end
+
+          it "redirects to /undertaker/morgue/HardWorker/RuntimeError/1_hour after the delete" do
+            subject
+            expect(last_response.status).to eq 302
+
+            # Redirect
+            follow_redirect!
+            expect(last_request.url).to eq expected_redirect_url
+            expect(last_response.status).to eq 200
+          end
+        end
+
+        context "when referer given" do
+          subject do
+            post("/undertaker/morgue",
+                 "key[]=#{job_refs[0]}&delete=Delete",
+                 "HTTP_REFERER" => "/undertaker/morgue/all/all/total_dead")
+          end
+
+          it "redirects back to referer after delete" do
+            subject
+            expect(last_response.status).to eq 302
+            expect(last_response.header["Location"]).to include "/undertaker/morgue/all/all/total_dead"
+          end
+        end
+
+        context "when /undertaker/morgue is called" do
+          context "when a key is given" do
+            subject { post "/undertaker/morgue", "key[]=#{job_refs[0]}&delete=Delete" }
+
+            it "deletes specific dead job now" do
+              expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(3)
+            end
+          end
+
+          context "when a key is missing" do
+            subject { post "/undertaker/morgue" }
+
+            it "returns 400 Bad Request" do
+              subject
+              expect(last_response.status).to eq 400
+            end
+          end
+        end
+      end
+
+      describe "retry" do
+        context "when job class, error and bucket are given" do
+          subject { post "/undertaker/morgue/HardWorker/RuntimeError/1_hour/retry" }
+
+          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+
+          let(:params) { { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour" } }
+          let(:dead_jobs_set) { [dead_job1, dead_job2] }
+          let(:dead_job1) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid1)) }
+          let(:dead_job2) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid2)) }
+
+          before do
+            allow(Sidekiq::Undertaker::JobFilter).to receive(:filter_dead_jobs).with(params)
+                                                                               .and_return(dead_jobs_set)
+          end
+
+          it "retries the dead jobs" do
+            expect(dead_job1.job).to receive(:retry).and_call_original
+            expect(dead_job2.job).to receive(:retry).and_call_original
+            subject
+          end
+
+          it "reduces DeadSet" do
+            expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(2)
+          end
+
+          it "redirects to /undertaker/morgue/HardWorker/RuntimeError/1_hour" do
+            subject
+            expect(last_response.status).to eq 302
+
+            # Redirect
+            follow_redirect!
+            expect(last_request.url).to eq expected_redirect_url
+            expect(last_response.status).to eq 200
+          end
+        end
+
+        context "when /undertaker/morgue is called" do
+          context "when a key is given" do
+            subject { post "/undertaker/morgue", "key[]=#{job_refs[0]}&retry=Retry+Now" }
+
+            it "reduces DeadSet" do
+              expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(3)
+            end
+          end
+
+          context "when a key is missing" do
+            subject { post "/undertaker/morgue" }
+
+            it "returns 400 Bad Request" do
+              subject
+              expect(last_response.status).to eq 400
+            end
+          end
+        end
+      end
+
+      describe "specific jobs" do
+        let(:dead_job) { Sidekiq::DeadSet.new.find_job(jid1) }
+
+        before do
+          score, jid = job_refs[0].split("-")
+
+          dead_set = instance_double(Sidekiq::DeadSet)
+
+          allow(dead_set).to receive(:fetch)
+            .with(score.to_f, jid)
+            .and_return([dead_job])
+
+          allow(Sidekiq::DeadSet).to receive(:new).and_return(dead_set)
+        end
+
+        it "retries specific dead job now" do
+          expect(dead_job).to receive(:retry)
+          post "/undertaker/morgue", "key[]=#{job_refs[0]}&retry=Retry+Now"
+        end
+
+        it "redirects on specific retry post" do
+          post("/undertaker/morgue",
+               "key[]=#{job_refs[0]}&retry=Retry+Now",
+               "HTTP_REFERER" => "/undertaker/morgue/all/all/total_dead")
+          expect(last_response.status).to eq 302
+          expect(last_response.header["Location"]).to include("/undertaker/morgue/all/all/total_dead")
+        end
+      end
+    end
+  end
+  # rubocop:enable Metrics/ModuleLength
+end