sidekiq-undertaker 1.2.0 → 1.4.1

data/spec/sidekiq/undertaker/job_filter_spec.rb

@@ -8,25 +8,28 @@ module Sidekiq
       describe ".filter_dead_jobs" do
         let(:job1) do
           instance_double(Sidekiq::JobRecord, item: {
-                            "class"       => "HardWorkTask",
-                            "failed_at"   => Time.now.to_i - 5 * 60,
-                            "error_class" => "NoMethodError"
+                            "class"         => "HardWorkTask",
+                            "failed_at"     => Time.now.to_i - (5 * 60),
+                            "error_class"   => "NoMethodError",
+                            "error_message" => "undefined method `pause` for H..."
                           })
         end
 
         let(:job2) do
           instance_double(Sidekiq::JobRecord, item: {
-                            "class"       => "HardWorkTask",
-                            "failed_at"   => Time.now.to_i - 2 * 60 * 60,
-                            "error_class" => "RandomError"
+                            "class"         => "HardWorkTask",
+                            "failed_at"     => Time.now.to_i - (2 * 60 * 60),
+                            "error_class"   => "RandomError",
+                            "error_message" => "random error message"
                           })
         end
 
         let(:job3) do
           instance_double(Sidekiq::JobRecord, item: {
-                            "class"       => "LazyWorkTask",
-                            "failed_at"   => Time.now.to_i - 2 * 60 * 60,
-                            "error_class" => "NoMethodError"
+                            "class"         => "LazyWorkTask",
+                            "failed_at"     => Time.now.to_i - (2 * 60 * 60),
+                            "error_class"   => "NoMethodError",
+                            "error_message" => "undefined method `work_hard` for LazyWork:Class"
                           })
         end
 
@@ -85,6 +88,20 @@ module Sidekiq
           it { expect(dead_jobs.size).to eq 2 }
         end
 
+        context "when the error_msg filter is given" do
+          subject(:dead_jobs) do
+            described_class.filter_dead_jobs("error_msg" => "undefined method `pause` for H...")
+          end
+
+          it "filters jobs based on error_message" do
+            dead_jobs.each do |dead_job|
+              expect(dead_job.error_msg).to eq "undefined method `pause` for H..."
+            end
+          end
+
+          it { expect(dead_jobs.size).to eq 1 }
+        end
+
         context "when no filters are applied" do
           subject(:dead_jobs) { described_class.filter_dead_jobs }
 

data/spec/sidekiq/undertaker/web_extension_spec.rb

@@ -6,6 +6,7 @@ require "sidekiq/api"
 require "sidekiq/web"
 require "sinatra"
 require "rack/test"
+require "stringio"
 
 module Sidekiq
   # rubocop:disable Metrics/ModuleLength
@@ -32,13 +33,15 @@ module Sidekiq
           "class"         => "HardWorker",
           "args"          => ["asdf", 1234],
           "queue"         => "foo",
-          "error_message" => "Some fake message",
+          "error_message" => "Option 'data/file_name' is required! This is an extra long error message.",
           "error_class"   => "RuntimeError",
           "retry_count"   => 0,
           "failed_at"     => Time.now.utc
         }
       end
 
+      let(:encoded_error_msg) { "T3B0aW9uICdkYXRhL2ZpbGVfbmFtZScgaXMgcmVxLi4u" }
+
       # rubocop:disable RSpec/AnyInstance
       before do
         Timecop.freeze(Time.gm(2018, 12, 16, 20, 57))
@@ -100,6 +103,20 @@ module Sidekiq
 
           it_behaves_like "a page"
         end
+
+        # /undertaker/filter/:job_class/:error_class/:bucket_name
+        context "when job-class/error-class/bucket page is called" do
+          subject { get "/undertaker/filter/HardWorker/RuntimeError/1_hour" }
+
+          it_behaves_like "a page"
+        end
+
+        # /undertaker/filter/:job_class/:error_class/:bucket_name?poll=true
+        context "when job-class/error-class/bucket page is polled" do
+          subject { get "/undertaker/filter/HardWorker/RuntimeError/1_hour?poll=true" }
+
+          it_behaves_like "a page"
+        end
       end
 
       describe "show morgue" do
@@ -107,10 +124,28 @@ module Sidekiq
           allow_any_instance_of(Sidekiq::Web::CsrfProtection).to receive(:mask_token).and_return("stubbed-csrf-token")
         end
 
-        # /undertaker/morgue/:job_class/:error_class/:bucket_name
+        # /undertaker/morgue/:job_class/:error_class/:error_msg/:bucket_name
         context "when job-class/error/bucket is called" do
           context "with specific job-class and a specific error" do
-            subject { get "/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+            subject { get "/undertaker/morgue/HardWorker/RuntimeError/all/1_hour" }
+
+            it_behaves_like "a page"
+
+            context "with pagination" do
+              before do
+                50.times do |i|
+                  job_refs.push add_dead("jid" => i.to_s)
+                end
+              end
+
+              it_behaves_like "a page"
+            end
+          end
+
+          context "with job-class, error and specific error message" do
+            subject do
+              get "/undertaker/morgue/HardWorker/RuntimeError/#{encoded_error_msg}/1_hour"
+            end
 
             it_behaves_like "a page"
 
@@ -126,7 +161,7 @@ module Sidekiq
           end
 
           context "with all failures and errors" do
-            subject { get "/undertaker/morgue/all/all/total_dead" }
+            subject { get "/undertaker/morgue/all/all/all/total_dead" }
 
             it_behaves_like "a page"
           end
@@ -135,12 +170,17 @@ module Sidekiq
       # rubocop:enable RSpec/AnyInstance
 
       describe "delete" do
-        context "when job-class, error and bucket are given" do
-          subject { post "/undertaker/morgue/HardWorker/RuntimeError/1_hour/delete" }
+        context "when job-class, error, error message and bucket are given" do
+          subject do
+            post "/undertaker/morgue/HardWorker/RuntimeError/#{encoded_error_msg}/1_hour/delete"
+          end
 
-          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/#{encoded_error_msg}/1_hour" }
 
-          let(:params) { { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour" } }
+          let(:params) do
+            { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour",
+           "error_msg" => "Option 'data/file_name' is req..." }
+          end
           let(:dead_jobs_set) { [dead_job1, dead_job2] }
           let(:dead_job1) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid1)) }
           let(:dead_job2) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid2)) }
@@ -160,7 +200,7 @@ module Sidekiq
             expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(2)
           end
 
-          it "redirects to /undertaker/morgue/HardWorker/RuntimeError/1_hour after the delete" do
+          it "redirects to morgue view after the delete" do
             subject
             expect(last_response.status).to eq 302
 
@@ -205,13 +245,77 @@ module Sidekiq
         end
       end
 
+      describe "import" do
+        subject { post "/undertaker/import_jobs", "upload_file" => file }
+
+        let(:file) do
+          Rack::Test::UploadedFile.new(StringIO.new(file_content), file_content_type, original_filename: file_name)
+        end
+        let(:job) do
+          opts = default_job_opts.merge({ "class" => "SuperHardWorking" })
+
+          build_job(opts)
+        end
+
+        context "when the file is valid" do
+          let(:file_content) { [job.item].to_json }
+          let(:file_name) { "jobs.json" }
+          let(:file_content_type) { "application/json" }
+
+          it "redirects the response" do
+            subject
+            expect(last_response.status).to eq 302
+          end
+
+          it "adds the jobs to the deadset" do
+            expect { subject }.to change { Sidekiq::DeadSet.new.size }.from(4).to(5)
+          end
+        end
+
+        context "when the file type is not valid" do
+          let(:file_content) { "" }
+          let(:file_name) { "jobs.zip" }
+          let(:file_content_type) { "application/zip" }
+
+          it "returns status 400" do
+            subject
+            expect(last_response.status).to eq 400
+          end
+        end
+
+        context "when the file type is a json but not a Sidekiq Job" do
+          let(:file_content) { "{am_i_a_job: \"no\"}" }
+          let(:file_name) { "jobs.json" }
+          let(:file_content_type) { "application/json" }
+
+          it "returns status 400" do
+            subject
+            expect(last_response.status).to eq 400
+          end
+        end
+
+        context "when the content of the file is not a json" do
+          let(:file_content) { "DEFINETLY NOT A JSON" }
+          let(:file_name) { "jobs.json" }
+          let(:file_content_type) { "application/json" }
+
+          it "returns status 400" do
+            subject
+            expect(last_response.status).to eq 400
+          end
+        end
+      end
+
       describe "retry" do
         context "when job class, error and bucket are given" do
-          subject { post "/undertaker/morgue/HardWorker/RuntimeError/1_hour/retry" }
+          subject { post "/undertaker/morgue/HardWorker/RuntimeError/all/1_hour/retry" }
 
-          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/1_hour" }
+          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/all/1_hour" }
 
-          let(:params) { { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour" } }
+          let(:params) do
+            { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour",
+              "error_msg" => "all" }
+          end
           let(:dead_jobs_set) { [dead_job1, dead_job2] }
           let(:dead_job1) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid1)) }
           let(:dead_job2) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid2)) }
@@ -262,6 +366,51 @@ module Sidekiq
         end
       end
 
+      describe "export" do
+        context "when job class, error and bucket are given" do
+          subject { post "/undertaker/morgue/HardWorker/RuntimeError/all/1_hour/export" }
+
+          let(:expected_redirect_url) { "http://example.org/undertaker/morgue/HardWorker/RuntimeError/all/1_hour" }
+          let(:expected_content_disposition_header) { "attachment; filename=\"2018-12-16_20-57.json\"" }
+
+          let(:params) do
+            { "job_class" => "HardWorker", "error_class" => "RuntimeError", "bucket_name" => "1_hour",
+              "error_msg" => "all" }
+          end
+          let(:dead_jobs_set) { [dead_job1, dead_job2] }
+          let(:dead_job1) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid1)) }
+          let(:dead_job2) { Sidekiq::Undertaker::DeadJob.to_dead_job(Sidekiq::DeadSet.new.find_job(jid2)) }
+
+          before do
+            allow(Sidekiq::Undertaker::JobFilter).to receive(:filter_dead_jobs).with(params)
+                                                                               .and_return(dead_jobs_set)
+          end
+
+          it "exports the dead jobs" do
+            subject
+            expect(last_response.status).to eq 200
+            expect(last_response.content_type).to eq "application/json"
+            expect(last_response.headers["Content-Disposition"]).to eq expected_content_disposition_header
+            expect(last_response.body).to eq dead_jobs_set.map { |t| t.job.item }.to_json
+          end
+
+          context "when there are more jobs than the current CHUNK_SIZE" do
+            before do
+              stub_const("Sidekiq::Undertaker::WebExtension::APIHelpers::EXPORT_CHUNK_SIZE", 1)
+            end
+
+            let(:expected_content_disposition_header) { "attachment; filename=\"HardWorker_2018-12-16_20-57.zip\"" }
+
+            it "exports the dead jobs" do
+              subject
+              expect(last_response.status).to eq 200
+              expect(last_response.content_type).to eq "application/zip"
+              expect(last_response.headers["Content-Disposition"]).to eq expected_content_disposition_header
+            end
+          end
+        end
+      end
+
       describe "specific jobs" do
         let(:dead_job) { Sidekiq::DeadSet.new.find_job(jid1) }
 
@@ -282,12 +431,21 @@ module Sidekiq
           post "/undertaker/morgue", "key[]=#{job_refs[0]}&retry=Retry+Now"
         end
 
+        it "exports specific dead job now" do
+          post "/undertaker/morgue", "key[]=#{job_refs[0]}&export=now"
+
+          expect(last_response.status).to eq 200
+          expect(last_response.content_type).to eq "application/json"
+          expect(last_response.headers["Content-Disposition"]).to eq "attachment; filename=\"2018-12-16_20-57.json\""
+          expect(last_response.body).to eq [dead_job.item].to_json
+        end
+
         it "redirects on specific retry post" do
           post("/undertaker/morgue",
                "key[]=#{job_refs[0]}&retry=Retry+Now",
-               "HTTP_REFERER" => "/undertaker/morgue/all/all/total_dead")
+               "HTTP_REFERER" => "/undertaker/morgue/all/all/all/total_dead")
           expect(last_response.status).to eq 302
-          expect(last_response.header["Location"]).to include("/undertaker/morgue/all/all/total_dead")
+          expect(last_response.header["Location"]).to include("/undertaker/morgue/all/all/all/total_dead")
         end
       end
     end

data/spec/spec_helper.rb

@@ -19,6 +19,10 @@ require "timecop"
 
 if ENV["COVERAGE"]
   require "simplecov"
+  require "simplecov_json_formatter"
+
+  SimpleCov.formatter = SimpleCov::Formatter::JSONFormatter
+
   SimpleCov.start do
     add_group "lib", "lib"
     add_group "spec", "spec"

data/web/locales/en.yml

@@ -3,3 +3,5 @@ en:
   UndertakerBuryAll:    Bury All
   UndertakerRevive:     Revive
   UndertakerReviveAll:  Revive All
+  UndertakerExport:     Export
+  UndertakerExportAll:  Export All

data/web/views/filter.erb

@@ -21,7 +21,7 @@
   </thead>
   <% @distribution.each do |group, bucket_counts| %>
       <tr>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{group}/all/total_dead" %>'> <%= group %></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{group}/all/all/total_dead" %>'> <%= group %></a></td>
         <td><a href='<%= "#{root_path}undertaker/filter/#{group}/total_dead" %>'><%= bucket_counts['total_dead']%></a></td>
         <td><a href='<%= "#{root_path}undertaker/filter/#{group}/1_hour" %>'><%= bucket_counts['1_hour']%></a></td>
         <td><a href='<%= "#{root_path}undertaker/filter/#{group}/3_hours" %>'><%= bucket_counts['3_hours']%></a></td>
@@ -32,3 +32,19 @@
       </tr>
   <% end %>
 </table>
+
+<header class="row header">
+  <div class="col-sm-12">
+    <h3>
+      Import Jobs
+    </h3>
+  </div>
+</header>
+
+<form enctype="multipart/form-data" method="post" action='<%="#{root_path}undertaker/import_jobs"%>'>
+  <%= respond_to?(:csrf_tag) && csrf_tag %>
+    <input type="file" id=upload_file" name="upload_file" >
+    <button class="btn btn-danger" style="margin-top: 10px" type="submit">
+      Import
+    </button>
+</form>

data/web/views/filter_job_class.erb

@@ -22,14 +22,14 @@
   </thead>
   <% @distribution.each do |group, bucket_counts| %>
       <tr>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/total_dead" %>'><%= group %></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/total_dead" %>'><%= bucket_counts['total_dead'] %></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/1_hour" %>'><%= bucket_counts['1_hour']%></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/3_hours" %>'><%= bucket_counts['3_hours']%></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/1_day" %>'><%= bucket_counts['1_day']%></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/3_days" %>'><%= bucket_counts['3_days']%></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/1_week" %>'><%= bucket_counts['1_week']%></a></td>
-        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/older" %>'><%= bucket_counts['older']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{group}/all/total_dead" %>'><%= group %></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/total_dead" %>'><%= bucket_counts['total_dead'] %></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/1_hour" %>'><%= bucket_counts['1_hour']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/3_hours" %>'><%= bucket_counts['3_hours']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/1_day" %>'><%= bucket_counts['1_day']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/3_days" %>'><%= bucket_counts['3_days']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/1_week" %>'><%= bucket_counts['1_week']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/filter/#{@req_job_class}/#{group}/older" %>'><%= bucket_counts['older']%></a></td>
       </tr>
   <% end %>
 </table>

data/web/views/filter_job_class_error_class.erb

@@ -0,0 +1,36 @@
+<header class="row header">
+  <div class="col-sm-12">
+    <h3>
+      <%= "<b>#{@total_dead}</b> dead #{@total_dead == 1 ? 'job' : 'jobs'}" %>
+      <%= " of <b>#{@req_job_class}</b> class" unless @req_job_class == "all" %>
+      <%= " with <b>#{@req_error_class}</b> exception" unless @req_error_class == "all" %>
+    </h3>
+  </div>
+</header>
+
+<table class="table table-striped table-bordered table-white">
+  <thead>
+    <tr>
+      <th style="width: 20%"><%= t('Error Message') %></th>
+      <th style="width: 10%"><%= t('All') %></th>
+      <th style="width: 10%"><%= t('1 hour') %></th>
+      <th style="width: 10%"><%= t('3 hours') %></th>
+      <th style="width: 10%"><%= t('1 day') %></th>
+      <th style="width: 10%"><%= t('3 days') %></th>
+      <th style="width: 10%"><%= t('1 week') %></th>
+      <th style="width: 10%"><%= t('Older') %></th>
+    </tr>
+  </thead>
+  <% @distribution.each do |group, bucket_counts| %>
+      <tr>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/total_dead" %>'><%= group %></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/total_dead" %>'><%= bucket_counts['total_dead'] %></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/1_hour" %>'><%= bucket_counts['1_hour']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/3_hours" %>'><%= bucket_counts['3_hours']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/1_day" %>'><%= bucket_counts['1_day']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/3_days" %>'><%= bucket_counts['3_days']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/1_week" %>'><%= bucket_counts['1_week']%></a></td>
+        <td><a href='<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(group)}/older" %>'><%= bucket_counts['older']%></a></td>
+      </tr>
+  <% end %>
+</table>

data/web/views/morgue.erb

@@ -4,6 +4,7 @@
       <%= "<b>#{@total_dead}</b> dead #{@total_dead == 1 ? 'job' : 'jobs'}" %>
       <%= " of <b>#{@req_job_class}</b> class" unless @req_job_class == "all" %>
       <%= " with <b>#{@req_error_class}</b> exception" unless @req_error_class == "all" %>
+      <%= " with <b>#{@req_error_msg}</b> message" unless @req_error_msg == "all" %>
       <%= " in <b>#{@req_bucket_name}</b> bucket"  %>
     </h3>
   </div>
@@ -56,15 +57,21 @@
         <% end %>
       </table>
       <input class="btn btn-primary btn-xs pull-left" type="submit" name="retry" value="<%= t('UndertakerRevive') %>" />
+      <input class="btn btn-secondary btn-xs pull-left" type="submit" name="export" value="<%= t('UndertakerExport') %>" />
       <input class="btn btn-danger btn-xs pull-left" type="submit" name="delete" value="<%= t('UndertakerBury') %>" />
     </form>
 
-    <form action="<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{@req_bucket_name}/delete" %>" method="post">
+    <form action="<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(@req_error_msg)}/#{@req_bucket_name}/delete" %>" method="post">
       <%= respond_to?(:csrf_tag) && csrf_tag %>
       <input class="btn btn-danger btn-xs pull-right" type="submit" name="delete" value="<%= t('UndertakerBuryAll') %>" data-confirm="<%= t('AreYouSure') %>" />
     </form>
 
-    <form action="<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{@req_bucket_name}/retry" %>" method="post">
+    <form action="<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(@req_error_msg)}/#{@req_bucket_name}/export" %>" method="post">
+      <%= respond_to?(:csrf_tag) && csrf_tag %>
+      <input class="btn btn-secondary btn-xs pull-right" type="submit" name="export" value="<%= t('UndertakerExportAll') %>" />
+    </form>
+
+    <form action="<%= "#{root_path}undertaker/morgue/#{@req_job_class}/#{@req_error_class}/#{Base64.urlsafe_encode64(@req_error_msg)}/#{@req_bucket_name}/retry" %>" method="post">
       <%= respond_to?(:csrf_tag) && csrf_tag %>
       <input class="btn btn-danger btn-xs pull-right" type="submit" name="retry" value="<%= t('UndertakerReviveAll') %>" data-confirm="<%= t('AreYouSure') %>" />
     </form>