sidekiq-error-groups 0.1.0

data/lib/sidekiq/error_groups/config.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    class Config
+      KNOWN_SETS = %i[retry dead].freeze
+      DEFAULT_SETS = KNOWN_SETS
+
+      # How many entries at most are read from each set during one aggregation.
+      # Scanning is linear in the size of the set, so this is the safety valve
+      # for installations with hundreds of thousands of dead jobs.
+      attr_accessor :max_scan
+
+      # How many matching jobs a single group action handles per pass.
+      attr_accessor :action_batch_size
+
+      # How many entries are listed in a group's breakdown column.
+      attr_accessor :top_breakdown
+
+      # Seconds to keep an aggregation result in process memory. 0 disables it.
+      attr_accessor :cache_ttl
+
+      # How many backtrace lines are shown for a group sample.
+      attr_accessor :backtrace_lines
+
+      # Jobs per page on the group detail page.
+      attr_accessor :per_page
+
+      attr_reader :sets, :group_by
+
+      def initialize
+        @max_scan = 10_000
+        @action_batch_size = 1_000
+        @top_breakdown = 3
+        @cache_ttl = 0
+        @backtrace_lines = 10
+        @per_page = 25
+        @sets = DEFAULT_SETS.dup
+        # Error class *and* job class: "Net::OpenTimeout in FetchInventoryJob" is
+        # a single actionable problem, "Net::OpenTimeout" alone usually is not.
+        @group_by = :error_job
+      end
+
+      def sets=(value)
+        @sets = self.class.normalize_sets(value)
+      end
+
+      def group_by=(value)
+        @group_by = self.class.normalize_group_by(value)
+      end
+
+      def self.normalize_sets(value)
+        sets = Array(value).map { |set| set.to_s.strip.to_sym }.uniq
+        unknown = sets - KNOWN_SETS
+        raise ArgumentError, "unknown job set(s): #{unknown.join(", ")}" if unknown.any?
+        raise ArgumentError, "at least one job set is required" if sets.empty?
+
+        sets
+      end
+
+      def self.normalize_group_by(value)
+        mode = value.to_s.strip.tr("-+ ", "___").to_sym
+        return mode if Selector::MODES.include?(mode)
+
+        raise ArgumentError, "unknown grouping: #{value}, expected one of #{Selector::MODES.join(", ")}"
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/formatter.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Plain text rendering of a {Summary}, shared by the CLI and the rake task.
+    module Formatter
+      NO_ERROR_CLASS = "(no error class)"
+
+      # Columns differ per grouping: the key is always shown, and whatever is not
+      # in the key becomes the breakdown column.
+      COLUMNS = {
+        error: ["ERROR CLASS", "TOTAL", "RETRY", "DEAD", "LAST FAILURE", "JOBS"],
+        job: ["JOB", "TOTAL", "RETRY", "DEAD", "LAST FAILURE", "ERRORS"],
+        error_job: ["ERROR CLASS", "JOB", "TOTAL", "RETRY", "DEAD", "LAST FAILURE"]
+      }.freeze
+
+      module_function
+
+      def render(summary, limit: nil, top_breakdown: nil)
+        lines = [headline(summary)]
+        return (lines << "" << "No failed jobs found.").join("\n") if summary.empty?
+
+        groups = limit ? summary.groups.first(limit) : summary.groups
+        lines << ""
+        lines.concat(table(groups, summary.group_by, top_breakdown: top_breakdown))
+        if limit && summary.groups.size > limit
+          lines << ""
+          lines << "... #{summary.groups.size - limit} more group(s), pass a bigger --limit to see them."
+        end
+        lines.join("\n")
+      end
+
+      def headline(summary)
+        parts = summary.set_sizes.map { |set, size| "#{set}: #{size}" }
+        line = "Failed jobs grouped by #{grouping_label(summary.group_by)} — #{summary.total} job(s) " \
+               "(#{parts.join(", ")}), #{summary.groups.size} group(s)"
+        line += "\nOnly the first #{summary.scanned} job(s) were scanned (max_scan), numbers are a lower bound." if summary.truncated?
+        line
+      end
+
+      def grouping_label(group_by)
+        {error: "error class", job: "job class", error_job: "error class and job class"}.fetch(group_by.to_sym)
+      end
+
+      def table(groups, group_by, top_breakdown: nil)
+        headers = COLUMNS.fetch(group_by.to_sym)
+        rows = groups.map { |group| row(group, group_by, top_breakdown) }
+        widths = headers.each_with_index.map do |header, index|
+          ([header] + rows.map { |r| r[index] }).map(&:length).max
+        end
+
+        separator = widths.map { |width| "-" * width }
+        ([headers, separator] + rows).map do |row|
+          row.each_with_index.map { |cell, index| cell.ljust(widths[index]) }.join("  ").rstrip
+        end
+      end
+
+      def row(group, group_by, top_breakdown)
+        key = case group_by.to_sym
+              when :error then [error_class_label(group.error_class)]
+              when :job then [group.job_class.to_s]
+              else [error_class_label(group.error_class), group.job_class.to_s]
+              end
+
+        counts = [group.count.to_s, group.retry_count.to_s, group.dead_count.to_s, timestamp(group.last_failed_at)]
+        return key + counts if group_by.to_sym == :error_job
+
+        key + counts + [breakdown_label(group, top_breakdown)]
+      end
+
+      def breakdown_label(group, top_breakdown)
+        limit = top_breakdown || ErrorGroups.config.top_breakdown
+        listed = group.breakdown(limit).map { |name, count| "#{error_class_label(name)} (#{count})" }
+        rest = group.breakdown_size - listed.size
+        listed << "+#{rest} more" if rest.positive?
+        listed.join(", ")
+      end
+
+      def error_class_label(error_class)
+        error_class == Selector::UNKNOWN ? NO_ERROR_CLASS : error_class.to_s
+      end
+
+      def timestamp(time)
+        return "-" unless time
+
+        time.strftime("%Y-%m-%d %H:%M:%S UTC")
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/group.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Aggregated numbers for one {Sidekiq::ErrorGroups::Selector}.
+    class Group
+      extend Forwardable
+
+      # Kept for backwards compatibility; the sentinel itself lives on Selector.
+      UNKNOWN = Selector::UNKNOWN
+
+      attr_reader :selector, :count, :set_counts, :job_classes, :error_classes,
+        :first_failed_at, :last_failed_at, :sample
+
+      def_delegators :selector, :error_class, :job_class, :mode, :matches?, :unknown_error_class?
+
+      def initialize(selector)
+        @selector = selector
+        @count = 0
+        @set_counts = {}
+        @job_classes = {}
+        @error_classes = {}
+        @first_failed_at = nil
+        @last_failed_at = nil
+        @sample = nil
+      end
+
+      def self.for(error_class: nil, job_class: nil)
+        new(Selector.new(error_class: error_class, job_class: job_class))
+      end
+
+      def add(job)
+        @count += 1
+        @set_counts[job.set] = count_for(job.set) + 1
+        tally(@job_classes, job.display_class)
+        tally(@error_classes, job.error_class)
+
+        track_timestamps(job)
+        track_sample(job)
+        self
+      end
+
+      def count_for(set)
+        @set_counts.fetch(set, 0)
+      end
+
+      def retry_count
+        count_for(:retry)
+      end
+
+      def dead_count
+        count_for(:dead)
+      end
+
+      def unknown?
+        unknown_error_class?
+      end
+
+      def top_job_classes(limit = ErrorGroups.config.top_breakdown)
+        top(job_classes, limit)
+      end
+
+      def top_error_classes(limit = ErrorGroups.config.top_breakdown)
+        top(error_classes, limit)
+      end
+
+      # The part of the group that is *not* in its key: grouping by error class
+      # leaves the job classes worth showing, and the other way around. Grouping
+      # by both leaves nothing to break down.
+      def breakdown(limit = ErrorGroups.config.top_breakdown)
+        case mode
+        when :error then top_job_classes(limit)
+        when :job then top_error_classes(limit)
+        else []
+        end
+      end
+
+      def breakdown_size
+        case mode
+        when :error then job_classes.size
+        when :job then error_classes.size
+        else 0
+        end
+      end
+
+      def to_h
+        {
+          error_class: error_class,
+          job_class: job_class,
+          count: count,
+          sets: set_counts,
+          job_classes: job_classes,
+          error_classes: error_classes,
+          first_failed_at: first_failed_at&.iso8601,
+          last_failed_at: last_failed_at&.iso8601,
+          sample: sample&.to_h
+        }.compact
+      end
+
+      private
+
+      def tally(counters, key)
+        counters[key] = counters.fetch(key, 0) + 1
+      end
+
+      def top(counters, limit)
+        counters.sort_by { |name, count| [-count, name] }.first(limit)
+      end
+
+      def track_timestamps(job)
+        first = job.failed_at
+        last = job.last_failure_at
+
+        @first_failed_at = first if first && (@first_failed_at.nil? || first < @first_failed_at)
+        @last_failed_at = last if last && (@last_failed_at.nil? || last > @last_failed_at)
+      end
+
+      # Keep the most recent failure as the sample: it is the one people want to
+      # look at when they open the group.
+      def track_sample(job)
+        return @sample = job if @sample.nil?
+
+        current = @sample.last_failure_at
+        candidate = job.last_failure_at
+        @sample = job if candidate && current && candidate > current
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/job.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "time"
+
+module Sidekiq
+  module ErrorGroups
+    # A failed job found in one of the job sets. Wraps a `Sidekiq::SortedEntry`
+    # and remembers which set it came from, so the UI can link back to the
+    # matching built-in page (`retries/...` or `morgue/...`).
+    #
+    # Only methods available in every supported Sidekiq (6.0+) are used, plus a
+    # few guarded with `respond_to?`.
+    class Job
+      WEB_SEGMENTS = {retry: "retries", dead: "morgue"}.freeze
+
+      attr_reader :set, :entry
+
+      def initialize(set, entry)
+        @set = set
+        @entry = entry
+      end
+
+      def item
+        @item ||= entry.item
+      end
+
+      def jid
+        item["jid"]
+      end
+
+      def queue
+        item["queue"]
+      end
+
+      def score
+        entry.score.to_f
+      end
+
+      def at
+        Time.at(score).utc
+      end
+
+      # Unwraps ActiveJob/Delayed wrappers when Sidekiq knows how.
+      def display_class
+        @display_class ||= begin
+          entry.display_class.to_s
+        rescue StandardError
+          item["class"].to_s
+        end
+      end
+
+      def error_class
+        value = item["error_class"]
+        value.nil? || value.to_s.empty? ? Selector::UNKNOWN : value.to_s
+      end
+
+      def error_message
+        item["error_message"].to_s
+      end
+
+      def retry_count
+        item["retry_count"].to_i
+      end
+
+      def failed_at
+        @failed_at ||= parse_time(item["failed_at"]) || at
+      end
+
+      def retried_at
+        @retried_at ||= parse_time(item["retried_at"])
+      end
+
+      # The most recent failure of this job: Sidekiq only updates `retried_at`
+      # on subsequent attempts.
+      def last_failure_at
+        retried_at || failed_at
+      end
+
+      # Sidekiq 6.5+ stores backtraces zlib+base64 encoded; `error_backtrace`
+      # decodes them. Older versions store a plain array.
+      def backtrace
+        return @backtrace if defined?(@backtrace)
+
+        @backtrace = begin
+          if entry.respond_to?(:error_backtrace)
+            entry.error_backtrace
+          else
+            raw = item["error_backtrace"]
+            raw.is_a?(Array) ? raw : nil
+          end
+        rescue StandardError
+          nil
+        end
+      end
+
+      def web_segment
+        WEB_SEGMENTS.fetch(set, "retries")
+      end
+
+      def retry!
+        entry.retry
+      end
+
+      def delete!
+        entry.delete
+      end
+
+      def kill!
+        entry.kill
+      end
+
+      def to_h
+        {
+          set: set,
+          jid: jid,
+          queue: queue,
+          job_class: display_class,
+          error_class: error_class,
+          error_message: error_message,
+          retry_count: retry_count,
+          failed_at: failed_at&.iso8601,
+          last_failure_at: last_failure_at&.iso8601
+        }
+      end
+
+      private
+
+      def parse_time(value)
+        case value
+        when Numeric then Time.at(value).utc
+        when String then Time.parse(value).utc rescue nil # rubocop:disable Style/RescueModifier
+        when Time then value.utc
+        end
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/job_query.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Paginated list of the jobs inside one group.
+    #
+    # Jobs come back in set order (retries before dead) and, within a set, in the
+    # order Sidekiq itself uses on its Retries/Dead pages: highest score first.
+    class JobQuery
+      Page = Struct.new(:jobs, :group, :total, :page, :per_page, :scanned, :truncated, keyword_init: true) do
+        def pages
+          return 1 if per_page.zero?
+
+          [(total.to_f / per_page).ceil, 1].max
+        end
+
+        def first_index
+          (page - 1) * per_page
+        end
+
+        def selector
+          group.selector
+        end
+      end
+
+      def self.call(error_class = nil, **kwargs)
+        new(error_class, **kwargs).call
+      end
+
+      def initialize(error_class = nil, job_class: nil, sets: nil, page: 1, per_page: nil, max_scan: nil, sources: nil)
+        @selector = Selector.build(error_class: error_class, job_class: job_class)
+        @sets = Config.normalize_sets(sets || ErrorGroups.config.sets)
+        @per_page = (per_page || ErrorGroups.config.per_page).to_i
+        @per_page = 25 if @per_page <= 0
+        @page = page.to_i
+        @page = 1 if @page <= 0
+        @max_scan = max_scan.nil? ? ErrorGroups.config.max_scan : max_scan
+        @sources = sources
+      end
+
+      def call
+        window = ((@page - 1) * @per_page)...(@page * @per_page)
+        matched = 0
+        jobs = []
+        # Accumulated from every matching job, not just the current page, so the
+        # detail view gets whole-group numbers out of a single scan.
+        group = Group.new(@selector)
+
+        scanned, sizes = Scanner.scan(sets: @sets, max_scan: @max_scan, sources: @sources) do |job|
+          next unless @selector.matches?(job)
+
+          jobs << job if window.cover?(matched)
+          group.add(job)
+          matched += 1
+        end
+
+        Page.new(
+          jobs: jobs,
+          group: group,
+          total: matched,
+          page: @page,
+          per_page: @per_page,
+          scanned: scanned,
+          truncated: scanned < sizes.values.sum
+        )
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/railtie.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Sidekiq
+  module ErrorGroups
+    class Railtie < ::Rails::Railtie
+      rake_tasks do
+        load File.expand_path("tasks.rake", __dir__)
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/scanner.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Walks the Sidekiq job sets and yields {Sidekiq::ErrorGroups::Job}s.
+    #
+    # Iteration goes through `Sidekiq::JobSet#each` on purpose: it paginates the
+    # sorted set internally and behaves identically on Sidekiq 6, 7 and 8.
+    # Talking to Redis directly would mean handling both redis-rb 4 (Sidekiq 6)
+    # and redis-client (Sidekiq 7+), which disagree on `ZRANGE ... WITHSCORES`.
+    module Scanner
+      module_function
+
+      # @return [Array(Integer, Hash)] number of entries read and size per set
+      def scan(sets:, max_scan: nil, sources: nil)
+        scanned = 0
+        sizes = {}
+
+        sets.each do |kind|
+          set = source_for(kind, sources)
+          sizes[kind] = size_of(set)
+          taken = 0
+
+          set.each do |entry|
+            break if max_scan && taken >= max_scan
+
+            taken += 1
+            scanned += 1
+            yield Job.new(kind, entry)
+          end
+        end
+
+        [scanned, sizes]
+      end
+
+      def source_for(kind, sources = nil)
+        return sources.fetch(kind) if sources
+
+        case kind.to_sym
+        when :retry then Sidekiq::RetrySet.new
+        when :dead then Sidekiq::DeadSet.new
+        else raise ArgumentError, "unknown job set: #{kind}"
+        end
+      end
+
+      def size_of(set)
+        set.size
+      rescue StandardError
+        0
+      end
+    end
+  end
+end

data/lib/sidekiq/error_groups/selector.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+module Sidekiq
+  module ErrorGroups
+    # Identifies one group: an error class, a job class, or the pair of them.
+    #
+    # The same object is the grouping key during aggregation, the filter used to
+    # list or act on a group's jobs, and the pair of query parameters the Web UI
+    # puts in its links.
+    class Selector
+      # Jobs that carry no `error_class` (or, in theory, no `class`) are still
+      # worth grouping. The sentinel keeps them addressable in URLs.
+      UNKNOWN = "_unknown_"
+
+      # Which part of a job the aggregator keys on.
+      MODES = %i[error job error_job].freeze
+
+      attr_reader :error_class, :job_class
+
+      def initialize(error_class: nil, job_class: nil)
+        @error_class = normalize(error_class)
+        @job_class = normalize(job_class)
+      end
+
+      # Builds the key a job belongs to under the given grouping mode.
+      def self.for(job, mode)
+        case mode
+        when :error then new(error_class: job.error_class)
+        when :job then new(job_class: job.display_class)
+        when :error_job then new(error_class: job.error_class, job_class: job.display_class)
+        else raise ArgumentError, "unknown grouping: #{mode}"
+        end
+      end
+
+      # Same as `new`, but refuses the empty selector, which would match every
+      # failed job — never what a caller asking for "this group" means.
+      def self.build(error_class: nil, job_class: nil)
+        selector = new(error_class: error_class, job_class: job_class)
+        raise ArgumentError, "an error class, a job class, or both are required" if selector.empty?
+
+        selector
+      end
+
+      def mode
+        return :error_job if error_class && job_class
+        return :error if error_class
+
+        :job if job_class
+      end
+
+      def empty?
+        error_class.nil? && job_class.nil?
+      end
+
+      def matches?(job)
+        (error_class.nil? || job.error_class == error_class) &&
+          (job_class.nil? || job.display_class == job_class)
+      end
+
+      def unknown_error_class?
+        error_class == UNKNOWN
+      end
+
+      def key
+        [error_class, job_class]
+      end
+      alias to_a key
+
+      def ==(other)
+        other.is_a?(Selector) && key == other.key
+      end
+      alias eql? ==
+
+      def hash
+        key.hash
+      end
+
+      def to_h
+        {error_class: error_class, job_class: job_class}.compact
+      end
+
+      # Query string pairs for the Web UI, and `--error`/`--job` for the CLI.
+      def to_params
+        to_h.transform_keys(&:to_s)
+      end
+
+      def to_s
+        [error_class, job_class].compact.join(" · ")
+      end
+
+      private
+
+      def normalize(value)
+        return nil if value.nil?
+
+        string = value.to_s
+        string.empty? ? UNKNOWN : string
+      end
+    end
+  end
+end