sidekiq-undertaker 1.1.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 00e0e140904b2a6c034ae86443f4097405c27089bd402faa9c567b9a4ed494ac
-  data.tar.gz: df53ecae31dbf8ab6535016bee735ab199eec205251e54332bd7e1f221c584f5
+  metadata.gz: c9faade9f819fcaff3bc2246d68197d085f6f90b10b82f38290f5a3cc5752f07
+  data.tar.gz: c08b3349474057a180eecf4cc64b0bb15f43112b264833bc6f9441b5ee139549
 SHA512:
-  metadata.gz: c91f3a2c28ac45514a4b2de47bfbbb9029bf1377daf88ae7aa589f807fb59e6d691fc717f6661041966561d76eb516de80e7cf52a9e8bb78a6c7ac8556b9ac5b
-  data.tar.gz: 57177fcdb514ab963e3b67917938b8571e16f0209156505fd1f20cc32bf1f787f2922245406f8263f3a3d990934bb81fc445acee89329b69ade09025815d2ae7
+  metadata.gz: 3d5e1d644890b7ef70348f41faf14961852f76f41dbd3ee9f3fd85a57fc29ee97a3476e2bf224a41f6e76ec17219e7e7ac04abcd6f569fee956883c0b72ad786
+  data.tar.gz: df1f8590c18238ed71a20d380510375f0eba41ea07e2cfd49d4f1e2f3bd0d2d8f3aad43b0bb07fcea9d44dbd242ef0551e93bc3b5091d470d39d3e12e86b0fcf

data/.github/workflows/ruby-build.yml

@@ -34,12 +34,15 @@ jobs:
           - '2.7'
           - 'jruby'
           - 'jruby-9.3.2.0'
-          - 'jruby-9.2.20.0'
+          - 'jruby-9.2.20.1'
           - 'truffleruby'
           - 'truffleruby+graalvm'
 
+    env:
+      JRUBY_OPTS: '--debug'
+
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Ruby
     # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby,
     # change this to (see https://github.com/ruby/setup-ruby#versioning):
@@ -48,4 +51,29 @@ jobs:
         ruby-version: ${{ matrix.ruby-version }}
         bundler-cache: true # runs 'bundle install' and caches installed gems automatically
     - name: Run tests
-      run: bundle exec rspec spec
+      run: COVERAGE=true bundle exec rspec
+    - name: 'Upload Coverage Report'
+      uses: actions/upload-artifact@v3
+      if: ${{ matrix.ruby-version == 'ruby' }}
+      with:
+        name: coverage-report
+        path: ./coverage
+        retention-days: 1
+
+  coverage:
+    needs: [ test ]
+    name: coverage
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Download Coverage Report
+      uses: actions/download-artifact@v3
+      with:
+        name: coverage-report
+        path: ./coverage
+    - uses: paambaati/codeclimate-action@v3.0.0
+      env:
+        # Set CC_TEST_REPORTER_ID as secret of your repo
+        CC_TEST_REPORTER_ID: ${{secrets.CC_TEST_REPORTER_ID}}
+      with:
+        debug: true

data/.rubocop.yml

@@ -7,3 +7,7 @@ inherit_from: .rubocop_todo.yml
 
 AllCops:
   TargetRubyVersion: 2.5
+
+Layout/LineLength:
+  Exclude:
+    - 'sidekiq-undertaker.gemspec'

data/.rubocop_codeclimate.yml

@@ -1,5 +1,14 @@
 require:
+  - rubocop-rake
   - rubocop-rspec
+
 inherit_from:
   - .rt_rubocop_defaults.yml
   - .rubocop_todo.yml
+
+AllCops:
+  TargetRubyVersion: 2.5
+
+Layout/LineLength:
+  Exclude:
+    - 'sidekiq-undertaker.gemspec'

data/CHANGELOG.md

@@ -2,6 +2,25 @@
 
 ## Unreleased
 
+## [1.4.0] - 2022-06-03
+### Added
+- Added a filter based on error messages
+
+### Changed
+- Updated rubocop to version 1.30
+- Updated rubyzip to version 2.3
+
+### Removed
+- Removed Travis CI integration
+
+## [1.3.0] - 2022-04-27
+### Added
+- Added option to export and re-import dead jobs
+
+## [1.2.0] - 2022-02-03
+### Changed
+- Updated sidekiq dependency to 6.4.0
+
 ## [1.1.1] - 2022-02-03
 ### Added
 - Added CHANGELOG.md

data/README.md

@@ -1,7 +1,7 @@
 # Sidekiq::Undertaker
 
 [![Gem Version](https://badge.fury.io/rb/sidekiq-undertaker.svg)](https://badge.fury.io/rb/sidekiq-undertaker)
-[![Build Status](https://travis-ci.org/ThomasKoppensteiner/sidekiq-undertaker.svg?branch=master)](https://travis-ci.org/ThomasKoppensteiner/sidekiq-undertaker)
+[![Ruby-Build](https://github.com/ThomasKoppensteiner/sidekiq-undertaker/actions/workflows/ruby-build.yml/badge.svg)](https://github.com/ThomasKoppensteiner/sidekiq-undertaker/actions/workflows/ruby-build.yml)
 [![Code Climate](https://codeclimate.com/github/ThomasKoppensteiner/sidekiq-undertaker.svg)](https://codeclimate.com/github/ThomasKoppensteiner/sidekiq-undertaker)
 [![Test Coverage](https://api.codeclimate.com/v1/badges/d442eb0a323d8911661f/test_coverage)](https://codeclimate.com/github/ThomasKoppensteiner/sidekiq-undertaker/test_coverage)
 [![Ruby Style Guide](https://img.shields.io/badge/code_style-rubocop-brightgreen.svg)](https://github.com/rubocop-hq/rubocop)
@@ -45,24 +45,24 @@ Or install it yourself as:
 
 The filter page shows a table with time-buckets as columns and rows for each job class.
 
-![Sidekiq Undertaker](Demo_Filter.png)
+![Sidekiq Undertaker](assets/Undertaker_demo_all_jobs.png)
 
 #### Job Filter View
 
 For each job class, you can drill down to view error distribution based on
 error class.
 
-![Sidekiq Undertaker](Demo_Job_Filter.png)
+![Sidekiq Undertaker](assets/Undertaker_demo_1_job_all_errors.png)
 
 #### Morgue View
 Finally, click on the individual error counts to display details of the
 errors in a list form.
 
-![Sidekiq Undertaker](Demo_Morgue_1_Job.png)
+![Sidekiq Undertaker](assets/Undertaker_demo_1_job_1_error.png)
 
 The morgue view can, for example, also show an error distribution over all job classes.
 
-![Sidekiq Undertaker](Demo_Morgue_all.png)
+![Sidekiq Undertaker](assets/Undertaker_demo_all_jobs_1_error.png)
 
 ## Contributing
 
@@ -81,7 +81,9 @@ this fork was renamed to `sidekiq-undertaker`.
 
 The [Sidekiq-Cleaner](https://github.com/HackingHabits/sidekiq-cleaner) gem was originally created by [Madan Thangavelu](https://github.com/HackingHabits).
 [Tout](https://github.com/Tout/sidekiq-cleaner) and [TheWudu](https://github.com/TheWudu/sidekiq-cleaner) also contributed to it.
-Thank you [Zlatko Rednjak](https://github.com/Rednjak) for adding the initial version of the `CHANGELOG.md`.
+[Zlatko Rednjak](https://github.com/Rednjak) added the initial version of the `CHANGELOG.md`.
+[Lucas Dell'Isola](https://github.com/ldellisola) added the export/import feature for dead jobs.
+[thoesl](https://github.com/thoesl) added the error message based filter.
 
 For the complete list of network members have a look at the [fork overview](https://github.com/ThomasKoppensteiner/sidekiq-undertaker/network/members).
 

data/lib/sidekiq/undertaker/dead_job.rb

@@ -27,7 +27,7 @@ module Sidekiq
         end
       end
 
-      attr_reader :job_class, :time_elapsed_since_failure, :error_class, :bucket_name, :job
+      attr_reader :job_class, :time_elapsed_since_failure, :error_class, :error_msg, :bucket_name, :job
 
       def initialize(args)
         @job                        = args.fetch(:job)
@@ -35,12 +35,14 @@ module Sidekiq
         @bucket_name                = args.fetch(:bucket_name)
         @job_class                  = args.fetch(:job_class, job.item["class"])
         @error_class                = args.fetch(:error_class, job.item["error_class"])
+        @error_msg                  = shorten_error_msg(args.fetch(:error_message, job.item["error_message"]).to_s)
       end
 
       def ==(other)
         job_class == other.job_class &&
           time_elapsed_since_failure == other.time_elapsed_since_failure &&
           error_class                == other.error_class &&
+          error_msg                  == other.error_msg &&
           bucket_name                == other.bucket_name &&
           job_eql?(other.job)
       end
@@ -48,7 +50,13 @@ module Sidekiq
 
       private
 
-      attr_writer :job_class, :time_elapsed_since_failure, :error_class, :bucket_name, :job
+      attr_writer :job_class, :time_elapsed_since_failure, :error_class, :error_message, :bucket_name, :job
+
+      def shorten_error_msg(msg)
+        max_error_msg_length = 30
+
+        msg.length > max_error_msg_length ? "#{msg[0, max_error_msg_length]}..." : msg
+      end
 
       def job_eql?(other_job) # rubocop:disable Metrics/AbcSize
         job.jid == other_job.jid &&

data/lib/sidekiq/undertaker/job_distributor.rb

@@ -19,6 +19,10 @@ module Sidekiq
         group_by(:error_class)
       end
 
+      def group_by_error_msg
+        group_by(:error_msg)
+      end
+
       private
 
       def distribute

data/lib/sidekiq/undertaker/job_filter.rb

@@ -7,7 +7,7 @@ module Sidekiq
     class JobFilter
       class << self
         def filter_dead_jobs(filters = {})
-          # filters can have keys in (job_class, error_class, bucket_name)
+          # filters can have keys in (job_class, error_class, error_msg, bucket_name)
           dead_jobs = []
           Sidekiq::Undertaker::DeadJob.for_each do |dead_job|
             filter_passed = true
@@ -26,7 +26,8 @@ module Sidekiq
           value.nil? ||
             total_dead_bucket?(filter, value) ||
             all_jobs?(filter, value) ||
-            all_errors?(filter, value)
+            all_errors?(filter, value) ||
+            all_error_msgs?(filter, value)
         end
 
         def total_dead_bucket?(filter, value)
@@ -40,6 +41,10 @@ module Sidekiq
         def all_errors?(filter, value)
           filter == "error_class" && value == "all"
         end
+
+        def all_error_msgs?(filter, value)
+          filter == "error_msg" && value == "all"
+        end
       end
     end
   end

data/lib/sidekiq/undertaker/version.rb

@@ -2,6 +2,6 @@
 
 module Sidekiq
   module Undertaker
-    VERSION = "1.1.1"
+    VERSION = "1.4.0"
   end
 end

data/lib/sidekiq/undertaker/web_extension/api_helpers.rb

@@ -2,15 +2,21 @@
 
 require "sidekiq/undertaker/job_distributor"
 require "sidekiq/undertaker/job_filter"
+require "base64"
+require "json"
+require "zip"
 
 module Sidekiq
   module Undertaker
     module WebExtension
+      # rubocop:disable Metrics/ModuleLength
       module APIHelpers
+        EXPORT_CHUNK_SIZE = 2000
+
         def show_filter
           store_request_params
 
-          @dead_jobs    = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(params)
+          @dead_jobs    = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
           @distribution = Sidekiq::Undertaker::JobDistributor.new(@dead_jobs).group_by_job_class
           @total_dead   = @dead_jobs.size
 
@@ -20,17 +26,27 @@ module Sidekiq
         def show_filter_by_job_class_bucket_name
           store_request_params
 
-          @dead_jobs    = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(params)
+          @dead_jobs    = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
           @distribution = Sidekiq::Undertaker::JobDistributor.new(@dead_jobs).group_by_error_class
           @total_dead   = @dead_jobs.size
 
           render_result("filter_job_class.erb")
         end
 
-        def show_undertaker_by_job_class_error_class_bucket_name
+        def show_filter_by_job_class_error_class_bucket_name
+          store_request_params
+
+          @dead_jobs    = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
+          @distribution = Sidekiq::Undertaker::JobDistributor.new(@dead_jobs).group_by_error_msg
+          @total_dead   = @dead_jobs.size
+
+          render_result("filter_job_class_error_class.erb")
+        end
+
+        def show_undertaker_by_job_class_error_class_error_msg_bucket_name
           store_request_params
 
-          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(params)
+          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
 
           # Display dead jobs as list
           @dead_jobs = @dead_jobs.map(&:job)
@@ -39,7 +55,7 @@ module Sidekiq
 
           # Pagination
           @total_dead   = @dead_jobs.size
-          @current_page = (params[:page] || 1).to_i
+          @current_page = (parsed_params[:page] || 1).to_i
           @count        = 50 # per page
           @dead_jobs    = @dead_jobs[((@current_page - 1) * @count), @count]
 
@@ -52,32 +68,36 @@ module Sidekiq
           params.delete("job_class")
           params.delete("bucket_name")
           params.delete("error_class")
+          params.delete("error_msg")
 
           render_result("morgue.erb")
         end
 
         def post_undertaker
-          raise ::ArgumentError.new("Key missing") unless params["key"]
-
-          params["key"].each do |key|
-            job = Sidekiq::DeadSet.new.fetch(*parse_params(key)).first
-            if job
-              if params["retry"]
-                job.retry
-              elsif params["delete"]
-                job.delete
-              end
-            end
-          end
+          raise ::ArgumentError.new("Key missing") unless parsed_params["key"]
 
-          redirect redirect_path(request)
+          jobs = parsed_params["key"].map { |k| Sidekiq::DeadSet.new.fetch(*parse_params(k)).first }.compact
+
+          handle_selected_jobs jobs
         rescue ::ArgumentError
           bad_request
         end
 
-        def post_undertaker_job_class_error_class_buckent_name_delete
+        def handle_selected_jobs(jobs)
+          return send_data(*prepare_data(jobs.map(&:item), EXPORT_CHUNK_SIZE)) if parsed_params["export"]
+
+          if parsed_params["retry"]
+            jobs.each(&:retry)
+          elsif parsed_params["delete"]
+            jobs.each(&:delete)
+          end
+
+          redirect redirect_path(request)
+        end
+
+        def post_undertaker_job_class_error_class_error_msg_bucket_name_delete
           store_request_params
-          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(params)
+          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
           @dead_jobs.each do |dead_job|
             dead_job.job.delete
           end
@@ -85,10 +105,10 @@ module Sidekiq
           redirect redirect_path(request)
         end
 
-        def post_undertaker_job_class_error_class_buckent_name_retry
+        def post_undertaker_job_class_error_class_error_msg_bucket_name_retry
           store_request_params
 
-          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(params)
+          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
           @dead_jobs.each do |dead_job|
             dead_job.job.retry
           end
@@ -96,14 +116,45 @@ module Sidekiq
           redirect redirect_path(request)
         end
 
+        def post_undertaker_job_class_error_class_error_msg_bucket_name_export
+          store_request_params
+
+          @dead_jobs = Sidekiq::Undertaker::JobFilter.filter_dead_jobs(parsed_params)
+          send_data(*prepare_data(@dead_jobs.map { |j| j.job.item }, EXPORT_CHUNK_SIZE))
+        end
+
+        def post_import_jobs
+          file = parsed_params["upload_file"]
+          raise ::ArgumentError.new("The file is not a json") if file.nil? || file[:type] != "application/json"
+
+          data = parsed_params["upload_file"][:tempfile].read
+          dead_set = Sidekiq::DeadSet.new
+
+          JSON.parse(data).each do |job|
+            dead_set.kill(Sidekiq.dump_json(job))
+          end
+          redirect redirect_path(request)
+        rescue StandardError
+          bad_request
+        end
+
         def render_result(template)
           render(:erb, File.read(File.join(view_path, template)))
         end
 
         def store_request_params
-          @req_job_class   = params["job_class"]
-          @req_bucket_name = params["bucket_name"]
-          @req_error_class = params["error_class"]
+          @req_job_class   = parsed_params["job_class"]
+          @req_bucket_name = parsed_params["bucket_name"]
+          @req_error_class = parsed_params["error_class"]
+          @req_error_msg   = parsed_params["error_msg"]
+        end
+
+        def parsed_params
+          @parsed_params ||= if params["error_msg"] && params["error_msg"] != "all"
+                               params.merge("error_msg" => Base64.urlsafe_decode64(params["error_msg"]))
+                             else
+                               params
+                             end
         end
 
         def view_path
@@ -112,13 +163,40 @@ module Sidekiq
 
         def redirect_path(request)
           path = request.referer ? URI.parse(request.referer).path : request.path
-          path.gsub("/delete", "").gsub("/retry", "")
+          path.gsub("/delete", "").gsub("/retry", "").gsub("/export", "")
+        end
+
+        def prepare_data(data, chunk_size)
+          filename = Time.now.strftime("%Y-%m-%d_%H-%M").to_s
+          return [data.to_json, "application/json", "#{filename}.json"] if data.length <= chunk_size
+
+          filename = "#{@req_job_class}_#{filename}"
+          zip = Zip::OutputStream.write_buffer do |file|
+            data.each_slice(chunk_size).each_with_index do |chunk, index|
+              file.put_next_entry("#{filename}_part-#{index + 1}.json")
+              file.write chunk.to_json
+            end
+          end
+
+          [zip.string, "application/zip", "#{filename}.zip"]
+        end
+
+        def send_data(data, content_type = "text/plain", file_name = "attachment.txt")
+          [
+            200,
+            {
+              "Content-Type"        => content_type,
+              "Content-Disposition" => "attachment; filename=\"#{file_name}\""
+            },
+            [data]
+          ]
         end
 
         def bad_request
           [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
         end
       end
+      # rubocop:enable Metrics/ModuleLength
     end
   end
 end

data/lib/sidekiq/undertaker/web_extension.rb

@@ -12,25 +12,34 @@ module Sidekiq
         app.get "/undertaker/filter" do
           show_filter
         end
-
         app.get "/undertaker/filter/:job_class/:bucket_name" do
           show_filter_by_job_class_bucket_name
         end
-
-        app.get "/undertaker/morgue/:job_class/:error_class/:bucket_name" do
-          show_undertaker_by_job_class_error_class_bucket_name
+        app.get "/undertaker/filter/:job_class/:error_class/:bucket_name" do
+          show_filter_by_job_class_error_class_bucket_name
+        end
+        app.get "/undertaker/morgue/:job_class/:error_class/:error_msg/:bucket_name" do
+          show_undertaker_by_job_class_error_class_error_msg_bucket_name
         end
 
         app.post "/undertaker/morgue" do
           post_undertaker
         end
 
-        app.post "/undertaker/morgue/:job_class/:error_class/:bucket_name/delete" do
-          post_undertaker_job_class_error_class_buckent_name_delete
+        app.post "/undertaker/morgue/:job_class/:error_class/:error_msg/:bucket_name/delete" do
+          post_undertaker_job_class_error_class_error_msg_bucket_name_delete
+        end
+
+        app.post "/undertaker/morgue/:job_class/:error_class/:error_msg/:bucket_name/retry" do
+          post_undertaker_job_class_error_class_error_msg_bucket_name_retry
+        end
+
+        app.post "/undertaker/morgue/:job_class/:error_class/:error_msg/:bucket_name/export" do
+          post_undertaker_job_class_error_class_error_msg_bucket_name_export
         end
 
-        app.post "/undertaker/morgue/:job_class/:error_class/:bucket_name/retry" do
-          post_undertaker_job_class_error_class_buckent_name_retry
+        app.post "/undertaker/import_jobs" do
+          post_import_jobs
         end
       end
       # rubocop:enable Metrics/MethodLength

data/sidekiq-undertaker.gemspec

@@ -21,8 +21,9 @@ Gem::Specification.new do |spec|
   spec.metadata      = {
     "homepage_uri"          => "https://github.com/ThomasKoppensteiner/sidekiq-undertaker",
     "source_code_uri"       => "https://github.com/ThomasKoppensteiner/sidekiq-undertaker",
+    "changelog_uri"         => "https://github.com/ThomasKoppensteiner/sidekiq-undertaker/blob/master/CHANGELOG.md",
     "bug_tracker_uri"       => "https://github.com/ThomasKoppensteiner/sidekiq-undertaker/issues",
-    "build_status_uri"      => "https://travis-ci.org/ThomasKoppensteiner/sidekiq-undertaker",
+    "build_status_uri"      => "https://github.com/ThomasKoppensteiner/sidekiq-undertaker/actions/workflows/ruby-build.yml",
     "rubygems_mfa_required" => "true"
   }
 
@@ -30,7 +31,6 @@ Gem::Specification.new do |spec|
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
-  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib", "lib/sidekiq/undertaker"]
 
   spec.add_development_dependency "bundler", ">= 1.17", "<3"
@@ -46,13 +46,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rspec-mocks", "~> 3.8"
   spec.add_development_dependency "rspec-sidekiq", "~> 3.0"
   spec.add_development_dependency "rt_rubocop_defaults", "~> 2.3"
-  spec.add_development_dependency "rubocop", "~> 1.8"
+  spec.add_development_dependency "rubocop", "~> 1.30"
   spec.add_development_dependency "rubocop-rake", "~> 0.5"
   spec.add_development_dependency "rubocop-rspec", "~> 2.0"
   spec.add_development_dependency "rubocop_runner", "~> 2.1"
-  spec.add_development_dependency "simplecov", "~> 0.14"
+  spec.add_development_dependency "simplecov", "~> 0.21"
   spec.add_development_dependency "sinatra", "~> 2.0"
   spec.add_development_dependency "timecop", "~> 0.9"
 
-  spec.add_runtime_dependency "sidekiq", ">= 6.2.2", "< 6.3"
+  spec.add_runtime_dependency "rubyzip", "~> 2.3"
+  spec.add_runtime_dependency "sidekiq", ">= 6.4", "< 7"
 end