sidekiq-error-groups 0.1.0

data/lib/sidekiq/error_groups/summary.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Result of one aggregation pass.
+    class Summary
+      attr_reader :groups, :group_by, :scanned, :set_sizes, :generated_at
+
+      def initialize(groups:, scanned:, set_sizes:, group_by: nil, generated_at: Time.now.utc)
+        @groups = groups
+        @group_by = group_by || ErrorGroups.config.group_by
+        @scanned = scanned
+        @set_sizes = set_sizes
+        @generated_at = generated_at
+      end
+
+      def total
+        set_sizes.values.sum
+      end
+
+      def sets
+        set_sizes.keys
+      end
+
+      def size_of(set)
+        set_sizes.fetch(set, 0)
+      end
+
+      # True when `max_scan` stopped us before the end of a set, i.e. the numbers
+      # below are a lower bound.
+      def truncated?
+        scanned < total
+      end
+
+      def empty?
+        groups.empty?
+      end
+
+      def find(error_class = nil, job_class: nil)
+        selector = Selector.new(error_class: error_class, job_class: job_class)
+        groups.find { |group| group.selector == selector }
+      end
+
+      def to_h
+        {
+          generated_at: generated_at.iso8601,
+          group_by: group_by,
+          scanned: scanned,
+          total: total,
+          truncated: truncated?,
+          sets: set_sizes,
+          groups: groups.map(&:to_h)
+        }
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/tasks.rake

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+namespace :sidekiq do
+  desc "Show failed Sidekiq jobs grouped by error class and/or job class " \
+       "(GROUP_BY=error_job SETS=retry,dead LIMIT=20 MAX_SCAN=10000)"
+  task error_groups: (defined?(Rails) ? [:environment] : []) do
+    require "sidekiq/error_groups"
+
+    summary = Sidekiq::ErrorGroups.summary(
+      sets: ENV["SETS"]&.split(","),
+      group_by: ENV.fetch("GROUP_BY", nil),
+      max_scan: ENV["MAX_SCAN"]&.to_i,
+      refresh: true
+    )
+    puts Sidekiq::ErrorGroups::Formatter.render(summary, limit: ENV["LIMIT"]&.to_i)
+  end
+end

data/lib/sidekiq/error_groups/version.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    VERSION = "0.1.0"
+  end
+end

data/lib/sidekiq/error_groups/web.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "web_extension"
+
+# Registers the "Error groups" tab in the Sidekiq Web UI.
+#
+# Require this where the Web UI is mounted (`config/routes.rb` in Rails, or the
+# rackup file), *not* in an initializer: Sidekiq worker processes should not load
+# the Web UI.
+#
+#   require "sidekiq/error_groups/web"
+#   mount Sidekiq::Web => "/sidekiq"
+if defined?(Sidekiq::Web)
+  extension = Sidekiq::ErrorGroups::WebExtension
+  tab = "ErrorGroups" # a locale key, see web/locales
+  index = "error_groups"
+
+  if Sidekiq::ErrorGroups::Compat.sidekiq_8?
+    Sidekiq::Web.configure do |config|
+      config.register(extension, name: index, tab: tab, index: index)
+    end
+  elsif Sidekiq::ErrorGroups::Compat.sidekiq_73?
+    Sidekiq::Web.register(extension, name: index, tab: tab, index: index)
+  else
+    Sidekiq::Web.register(extension)
+    Sidekiq::Web.tabs[tab] = index
+  end
+end

data/lib/sidekiq/error_groups/web_extension.rb

@@ -0,0 +1,178 @@
+# frozen_string_literal: true
+
+require "erb"
+require_relative "../error_groups"
+
+module Sidekiq
+  module ErrorGroups
+    # Routes and view helpers for the "Error groups" tab.
+    #
+    # Registered through {Sidekiq::ErrorGroups::Web}, which knows how to talk to
+    # every supported Sidekiq release. Everything used here (`app.get`,
+    # `app.post`, `app.helpers`, `erb(string)`, `root_path`, `csrf_tag`, `t`,
+    # `relative_time`, `job_params`, `h`) exists in Sidekiq 6.0 through 8.x.
+    #
+    # A group is addressed by query parameters rather than path segments: the key
+    # can hold two class names, and `error_class` + `job_class` survive every
+    # Sidekiq router version unchanged.
+    module WebExtension
+      WEB_ROOT = File.expand_path("../../../web", __dir__)
+      NOTICES = %w[retry delete kill].freeze
+
+      module Helpers
+        # Sidekiq 8 turned both param readers into methods; before that they were
+        # hashes.
+        def eg_route_param(key)
+          Compat.sidekiq_8? ? route_params(key) : route_params[key]
+        end
+
+        def eg_url_param(key)
+          value = Compat.sidekiq_8? ? url_params(key.to_s) : params[key.to_s]
+          value.nil? || value.to_s.empty? ? nil : value.to_s
+        end
+
+        # Sidekiq 8 ships a completely different UI markup, so each view exists
+        # twice: `web/views` for 8.x and `web/views/legacy` for 6.x/7.x.
+        def eg_render(view)
+          dir = Compat.sidekiq_8? ? "views" : File.join("views", "legacy")
+          erb(File.read(File.join(WEB_ROOT, dir, "#{view}.erb")))
+        end
+
+        def eg_root
+          "#{root_path}error_groups"
+        end
+
+        def eg_group_by
+          @eg_group_by ||= begin
+            Config.normalize_group_by(eg_url_param(:group_by) || ErrorGroups.config.group_by)
+          rescue ArgumentError
+            ErrorGroups.config.group_by
+          end
+        end
+
+        def eg_index_path(group_by = eg_group_by, extra = {})
+          eg_path(eg_root, {"group_by" => group_by.to_s}.merge(extra))
+        end
+
+        def eg_group_path(selector, action = nil, extra = {})
+          base = action ? "#{eg_root}/group/#{action}" : "#{eg_root}/group"
+          eg_path(base, selector.to_params.merge(extra))
+        end
+
+        def eg_page_path(selector, page)
+          eg_group_path(selector, nil, {"page" => page.to_s})
+        end
+
+        def eg_path(base, params)
+          query = params.reject { |_, value| value.nil? || value.to_s.empty? }
+                        .map { |key, value| "#{ERB::Util.url_encode(key)}=#{ERB::Util.url_encode(value)}" }
+          query.empty? ? base : "#{base}?#{query.join("&")}"
+        end
+
+        def eg_label(error_class)
+          error_class == Selector::UNKNOWN ? t("NoErrorClass") : error_class.to_s
+        end
+
+        def eg_selector_label(selector)
+          [
+            selector.error_class && eg_label(selector.error_class),
+            selector.job_class
+          ].compact.join(" · ")
+        end
+
+        def eg_group_modes
+          Selector::MODES.map { |mode| [mode, t("GroupBy_#{mode}")] }
+        end
+
+        # Link to the job on Sidekiq's own Retries/Dead detail page.
+        def eg_job_path(job)
+          "#{root_path}#{job.web_segment}/#{job_params(job.item, job.score)}"
+        end
+
+        def eg_time(time)
+          return "-" unless time
+
+          relative_time(time)
+        end
+
+        def eg_breakdown(group)
+          group.breakdown.map { |name, count| "#{eg_label(name)} (#{count})" }.join(", ")
+        end
+
+        def eg_backtrace(job)
+          lines = job&.backtrace
+          lines ? lines.first(ErrorGroups.config.backtrace_lines) : []
+        end
+
+        # Hidden fields that keep a bulk action scoped to the group it was fired
+        # from, and bring the viewer back to the grouping they were looking at.
+        def eg_selector_fields(selector)
+          fields = selector.to_params.merge("group_by" => eg_group_by.to_s)
+          fields.map do |name, value|
+            "<input type='hidden' name='#{h name}' value='#{h value}'/>"
+          end.join
+        end
+
+        # Flash replacement: Sidekiq's `flash` helper needs a session and only
+        # exists on 8.x, so the result of a bulk action travels in the query
+        # string instead.
+        def eg_notice
+          operation = eg_url_param(:done)
+          return nil unless NOTICES.include?(operation)
+
+          count = eg_url_param(:count).to_i
+          group = eg_url_param(:group).to_s
+          t("GroupAction_#{operation}", count: count, group: group)
+        end
+      end
+
+      def self.registered(app)
+        locales(app) << File.join(WEB_ROOT, "locales")
+        app.helpers(Helpers)
+
+        app.get "/error_groups" do
+          @summary = Sidekiq::ErrorGroups.summary(group_by: eg_group_by)
+          eg_render(:index)
+        end
+
+        app.get "/error_groups/group" do
+          @page = Sidekiq::ErrorGroups.jobs(
+            eg_url_param(:error_class),
+            job_class: eg_url_param(:job_class),
+            page: eg_url_param(:page) || 1
+          )
+          @group = @page.group
+          @selector = @group.selector
+          eg_render(:group)
+        rescue ArgumentError
+          redirect eg_index_path
+        end
+
+        NOTICES.each do |operation|
+          app.post "/error_groups/group/#{operation}" do
+            selector = Selector.build(
+              error_class: eg_url_param(:error_class),
+              job_class: eg_url_param(:job_class)
+            )
+            result = Sidekiq::ErrorGroups::Actions.call(
+              selector.error_class,
+              job_class: selector.job_class,
+              operation: operation.to_sym
+            )
+
+            redirect eg_index_path(eg_group_by,
+              "done" => operation, "count" => result.total.to_s, "group" => eg_selector_label(selector))
+          rescue ArgumentError
+            redirect eg_index_path
+          end
+        end
+      end
+
+      # Where to append our locale directory. Sidekiq 8 moved the registry from
+      # the application class onto the Web config object.
+      def self.locales(app)
+        Compat.sidekiq_8? ? Sidekiq::Web.configure.locales : app.settings.locales
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+require "sidekiq"
+require "sidekiq/api"
+
+require_relative "error_groups/version"
+require_relative "error_groups/compat"
+require_relative "error_groups/selector"
+require_relative "error_groups/config"
+require_relative "error_groups/job"
+require_relative "error_groups/group"
+require_relative "error_groups/summary"
+require_relative "error_groups/scanner"
+require_relative "error_groups/aggregator"
+require_relative "error_groups/job_query"
+require_relative "error_groups/actions"
+require_relative "error_groups/formatter"
+
+module Sidekiq
+  # Groups failed Sidekiq jobs (retries + dead set) by error class, by job class,
+  # or by the pair of them.
+  #
+  #   Sidekiq::ErrorGroups.summary.groups.each do |group|
+  #     puts "#{group.error_class} in #{group.job_class}: #{group.count}"
+  #   end
+  module ErrorGroups
+    class Error < StandardError; end
+
+    class << self
+      def config
+        @config ||= Config.new
+      end
+
+      def configure
+        yield config if block_given?
+        config
+      end
+
+      # @return [Sidekiq::ErrorGroups::Summary]
+      def summary(sets: nil, group_by: nil, max_scan: nil, sources: nil, refresh: false)
+        options = {sets: sets, group_by: group_by, max_scan: max_scan, sources: sources}
+        ttl = config.cache_ttl.to_f
+        return Aggregator.call(**options) if sources || ttl <= 0
+
+        key = [
+          Config.normalize_sets(sets || config.sets),
+          Config.normalize_group_by(group_by || config.group_by),
+          max_scan || config.max_scan
+        ]
+
+        cache_mutex.synchronize do
+          entry = cache[key]
+          return entry[:summary] if !refresh && entry && entry[:expires_at] > monotonic_now
+
+          summary = Aggregator.call(**options)
+          cache[key] = {summary: summary, expires_at: monotonic_now + ttl}
+          summary
+        end
+      end
+
+      # @return [Array<Sidekiq::ErrorGroups::Group>]
+      def groups(**kwargs)
+        summary(**kwargs).groups
+      end
+
+      # @return [Sidekiq::ErrorGroups::JobQuery::Page]
+      def jobs(error_class = nil, **kwargs)
+        JobQuery.call(error_class, **kwargs)
+      end
+
+      def retry_all(error_class = nil, **kwargs)
+        Actions.retry_all(error_class, **kwargs)
+      end
+
+      def delete_all(error_class = nil, **kwargs)
+        Actions.delete_all(error_class, **kwargs)
+      end
+
+      def kill_all(error_class = nil, **kwargs)
+        Actions.kill_all(error_class, **kwargs)
+      end
+
+      def job_set(kind)
+        Scanner.source_for(kind)
+      end
+
+      def reset_cache!
+        cache_mutex.synchronize { cache.clear }
+      end
+
+      # @api private
+      def reset!
+        @config = nil
+        reset_cache!
+      end
+
+      private
+
+      def cache
+        @cache ||= {}
+      end
+
+      def cache_mutex
+        @cache_mutex ||= Mutex.new
+      end
+
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end
+
+require_relative "error_groups/railtie" if defined?(Rails::Railtie)

data/lib/sidekiq-error-groups.rb

@@ -0,0 +1,3 @@
+# frozen_string_literal: true
+
+require "sidekiq/error_groups"

data/sidekiq-error-groups.gemspec

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "lib/sidekiq/error_groups/version"
+
+Gem::Specification.new do |spec|
+  spec.name = "sidekiq-error-groups"
+  spec.version = Sidekiq::ErrorGroups::VERSION
+  spec.authors = ["r3cha"]
+  spec.email = ["frontandstart@gmail.com"]
+
+  spec.summary = "Sidekiq Web UI tab with failed jobs grouped by error class."
+  spec.description = "Adds an \"Error groups\" tab to the Sidekiq Web UI that groups retrying and " \
+                     "dead jobs by their error class, with per-group retry/delete/kill actions, " \
+                     "plus a Ruby API, a rake task and a CLI. Works with Sidekiq 6, 7 and 8."
+  spec.homepage = "https://github.com/r3cha/sidekiq-error-groups"
+  spec.license = "MIT"
+  spec.required_ruby_version = ">= 2.7.0"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = spec.homepage
+  spec.metadata["changelog_uri"] = "#{spec.homepage}/blob/main/CHANGELOG.md"
+  spec.metadata["bug_tracker_uri"] = "#{spec.homepage}/issues"
+  spec.metadata["documentation_uri"] = "#{spec.homepage}#readme"
+  spec.metadata["rubygems_mfa_required"] = "true"
+
+  spec.files = Dir.glob("lib/**/*") + Dir.glob("web/**/*") + %w[
+    CHANGELOG.md
+    LICENSE.txt
+    README.md
+    sidekiq-error-groups.gemspec
+  ]
+  spec.bindir = "exe"
+  spec.executables = ["sidekiq-error-groups"]
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "sidekiq", ">= 6.0"
+end

data/web/locales/en.yml

@@ -0,0 +1,35 @@
+en:
+  ErrorGroups: Error groups
+  ErrorClass: Error class
+  JobClass: Job
+  ErrorMessage: Error message
+  Backtrace: Backtrace
+  Total: Total
+  Attempts: Attempts
+  Set: Set
+  GroupBy: Group by
+  GroupBy_error: error class
+  GroupBy_job: job
+  GroupBy_error_job: error class + job
+  AffectedJobs: Job classes
+  AffectedErrors: Error classes
+  FirstFailure: First failure
+  LastFailure: Last failure
+  NoErrorClass: "(no error class)"
+  NoFailedJobs: No failed jobs. Nothing to group.
+  GroupEmpty: No jobs left in this group.
+  BackToErrorGroups: All error groups
+  ScannedJobs: "Scanned %{scanned} of %{total} failed job(s)"
+  ScanTruncated: "Scanning stopped at max_scan (%{max_scan}), the numbers below are a lower bound."
+  RetryGroup: Retry all in group
+  DeleteGroup: Delete all in group
+  KillGroup: Kill all in group
+  AreYouSureRetryGroup: "Are you sure you want to retry %{count} job(s)?"
+  AreYouSureDeleteGroup: "Are you sure you want to delete %{count} job(s)?"
+  AreYouSureKillGroup: "Are you sure you want to move %{count} job(s) to the dead set?"
+  GroupAction_retry: "Retried %{count} job(s) of %{group}"
+  GroupAction_delete: "Deleted %{count} job(s) of %{group}"
+  GroupAction_kill: "Moved %{count} job(s) of %{group} to the dead set"
+  PreviousPage: Previous
+  NextPage: Next
+  PageOf: "page %{page} of %{pages}"

data/web/locales/ru.yml

@@ -0,0 +1,35 @@
+ru:
+  ErrorGroups: Группы ошибок
+  ErrorClass: Класс ошибки
+  JobClass: Джоба
+  ErrorMessage: Сообщение об ошибке
+  Backtrace: Трейс
+  Total: Всего
+  Attempts: Попыток
+  Set: Набор
+  GroupBy: Группировать по
+  GroupBy_error: классу ошибки
+  GroupBy_job: джобе
+  GroupBy_error_job: ошибке и джобе
+  AffectedJobs: Классы джоб
+  AffectedErrors: Классы ошибок
+  FirstFailure: Первое падение
+  LastFailure: Последнее падение
+  NoErrorClass: "(без класса ошибки)"
+  NoFailedJobs: Упавших джоб нет — группировать нечего.
+  GroupEmpty: В этой группе больше нет джоб.
+  BackToErrorGroups: Все группы ошибок
+  ScannedJobs: "Просмотрено %{scanned} из %{total} упавших джоб"
+  ScanTruncated: "Просмотр остановлен на max_scan (%{max_scan}), числа ниже — нижняя оценка."
+  RetryGroup: Перезапустить всю группу
+  DeleteGroup: Удалить всю группу
+  KillGroup: Отправить группу в dead
+  AreYouSureRetryGroup: "Перезапустить %{count} джоб(ы)?"
+  AreYouSureDeleteGroup: "Удалить %{count} джоб(ы)?"
+  AreYouSureKillGroup: "Перенести %{count} джоб(ы) в dead-набор?"
+  GroupAction_retry: "Перезапущено джоб: %{count} (%{group})"
+  GroupAction_delete: "Удалено джоб: %{count} (%{group})"
+  GroupAction_kill: "Перенесено в dead: %{count} (%{group})"
+  PreviousPage: Назад
+  NextPage: Вперёд
+  PageOf: "страница %{page} из %{pages}"

data/web/views/group.erb

@@ -0,0 +1,114 @@
+<section>
+  <header>
+    <h1><%= h eg_selector_label(@selector) %></h1>
+    <p><a href="<%= h eg_index_path %>">&larr; <%= t('BackToErrorGroups') %></a></p>
+  </header>
+
+  <% if @group.count.zero? %>
+    <p><%= t('GroupEmpty') %></p>
+  <% else %>
+    <div class="table_container">
+      <table>
+        <tbody>
+          <% if @selector.error_class %>
+            <tr><th><%= t('ErrorClass') %></th><td><%= h eg_label(@selector.error_class) %></td></tr>
+          <% end %>
+          <% if @selector.job_class %>
+            <tr><th><%= t('JobClass') %></th><td><%= h @selector.job_class %></td></tr>
+          <% end %>
+          <tr><th><%= t('Total') %></th><td><%= @group.count %></td></tr>
+          <tr><th><%= t('Retries') %></th><td><%= @group.retry_count %></td></tr>
+          <tr><th><%= t('Dead') %></th><td><%= @group.dead_count %></td></tr>
+          <tr><th><%= t('FirstFailure') %></th><td><%= eg_time(@group.first_failed_at) %></td></tr>
+          <tr><th><%= t('LastFailure') %></th><td><%= eg_time(@group.last_failed_at) %></td></tr>
+          <% unless @group.breakdown.empty? %>
+            <tr>
+              <th><%= (@group.mode == :error) ? t('AffectedJobs') : t('AffectedErrors') %></th>
+              <td><%= h eg_breakdown(@group) %></td>
+            </tr>
+          <% end %>
+          <% if @group.sample && !@group.sample.error_message.empty? %>
+            <tr><th><%= t('ErrorMessage') %></th><td><code><%= h @group.sample.error_message %></code></td></tr>
+          <% end %>
+          <% backtrace = eg_backtrace(@group.sample) %>
+          <% unless backtrace.empty? %>
+            <tr>
+              <th><%= t('Backtrace') %></th>
+              <td><code><%= backtrace.map { |line| h(line) }.join("<br/>") %></code></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+
+    <div class="buttons-row">
+      <form action="<%= h eg_group_path(@selector, 'retry') %>" method="post">
+        <%= csrf_tag %>
+        <%= eg_selector_fields(@selector) %>
+        <input class="btn btn-primary" type="submit" value="<%= t('RetryGroup') %>"
+               data-confirm="<%= t('AreYouSureRetryGroup', count: @group.count) %>" />
+      </form>
+      <% if @group.retry_count.positive? %>
+        <form action="<%= h eg_group_path(@selector, 'kill') %>" method="post">
+          <%= csrf_tag %>
+          <%= eg_selector_fields(@selector) %>
+          <input class="btn btn-danger" type="submit" value="<%= t('KillGroup') %>"
+                 data-confirm="<%= t('AreYouSureKillGroup', count: @group.retry_count) %>" />
+        </form>
+      <% end %>
+      <form action="<%= h eg_group_path(@selector, 'delete') %>" method="post">
+        <%= csrf_tag %>
+        <%= eg_selector_fields(@selector) %>
+        <input class="btn btn-danger" type="submit" value="<%= t('DeleteGroup') %>"
+               data-confirm="<%= t('AreYouSureDeleteGroup', count: @group.count) %>" />
+      </form>
+    </div>
+
+    <% if @page.truncated %>
+      <p><strong><%= t('ScanTruncated', max_scan: Sidekiq::ErrorGroups.config.max_scan) %></strong></p>
+    <% end %>
+
+    <div class="table_container">
+      <table>
+        <thead>
+          <tr>
+            <th><%= t('Set') %></th>
+            <th><%= t('LastFailure') %></th>
+            <th><%= t('Queue') %></th>
+            <th><%= t('Job') %></th>
+            <% unless @selector.error_class %>
+              <th><%= t('ErrorClass') %></th>
+            <% end %>
+            <th><%= t('Arguments') %></th>
+            <th><%= t('Attempts') %></th>
+          </tr>
+        </thead>
+        <tbody>
+          <% @page.jobs.each do |job| %>
+            <tr>
+              <td><%= h job.set.to_s %></td>
+              <td><a href="<%= eg_job_path(job) %>"><%= eg_time(job.last_failure_at) %></a></td>
+              <td><a href="<%= root_path %>queues/<%= h job.queue.to_s %>"><%= h job.queue.to_s %></a></td>
+              <td><%= h job.display_class %></td>
+              <% unless @selector.error_class %>
+                <td><%= h eg_label(job.error_class) %></td>
+              <% end %>
+              <td><%= display_args(job.item["args"]) %></td>
+              <td><%= job.retry_count %></td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    </div>
+
+    <p>
+      <% if @page.page > 1 %>
+        <a href="<%= h eg_page_path(@selector, @page.page - 1) %>">&larr; <%= t('PreviousPage') %></a>
+      <% end %>
+      <%= t('PageOf', page: @page.page, pages: @page.pages) %>
+      <% if @page.page < @page.pages %>
+        <a href="<%= h eg_page_path(@selector, @page.page + 1) %>"><%= t('NextPage') %> &rarr;</a>
+      <% end %>
+    </p>
+  <% end %>
+</section>