sidekiq-error-groups 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +9 -0
- data/LICENSE.txt +21 -0
- data/README.md +238 -0
- data/exe/sidekiq-error-groups +6 -0
- data/lib/sidekiq/error_groups/actions.rb +124 -0
- data/lib/sidekiq/error_groups/aggregator.rb +40 -0
- data/lib/sidekiq/error_groups/cli.rb +155 -0
- data/lib/sidekiq/error_groups/compat.rb +36 -0
- data/lib/sidekiq/error_groups/config.rb +69 -0
- data/lib/sidekiq/error_groups/formatter.rb +90 -0
- data/lib/sidekiq/error_groups/group.rb +129 -0
- data/lib/sidekiq/error_groups/job.rb +137 -0
- data/lib/sidekiq/error_groups/job_query.rb +69 -0
- data/lib/sidekiq/error_groups/railtie.rb +13 -0
- data/lib/sidekiq/error_groups/scanner.rb +53 -0
- data/lib/sidekiq/error_groups/selector.rb +101 -0
- data/lib/sidekiq/error_groups/summary.rb +57 -0
- data/lib/sidekiq/error_groups/tasks.rake +17 -0
- data/lib/sidekiq/error_groups/version.rb +7 -0
- data/lib/sidekiq/error_groups/web.rb +28 -0
- data/lib/sidekiq/error_groups/web_extension.rb +178 -0
- data/lib/sidekiq/error_groups.rb +116 -0
- data/lib/sidekiq-error-groups.rb +3 -0
- data/sidekiq-error-groups.gemspec +37 -0
- data/web/locales/en.yml +35 -0
- data/web/locales/ru.yml +35 -0
- data/web/views/group.erb +114 -0
- data/web/views/index.erb +83 -0
- data/web/views/legacy/group.erb +114 -0
- data/web/views/legacy/index.erb +83 -0
- metadata +93 -0
checksums.yaml
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
---
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: 62c658e2afd9ef643352622f3298122194db22fbe9a620e528d436328b84370e
|
|
4
|
+
data.tar.gz: 8646512a57ae85bbc03e4717f39dba749e4ebb0d372bbd213383a417e8fd0552
|
|
5
|
+
SHA512:
|
|
6
|
+
metadata.gz: 4464480cf1b46762345ee2f01691528d9ac6601872af14b3f3f5a6333f047f9512c36e940388578d2b2c9f7706bad9e122887db10cc3ae13dcd90edbaa135b66
|
|
7
|
+
data.tar.gz: a1f7c3243b44bb54c9f317e2cd946aa5690c20d4a1531ca8924d2728a94ae48a58bad8094d14e447912c3ff5fb0931a6f8345e508ef2b44ff0fe698e18597fe9
|
data/CHANGELOG.md
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
## [Unreleased]
|
|
2
|
+
|
|
3
|
+
## [0.1.0] - 2026-10-06
|
|
4
|
+
|
|
5
|
+
- Initial release: "Error groups" tab for the Sidekiq Web UI (Sidekiq 6, 7 and 8).
|
|
6
|
+
- Group failed jobs by error class, by job class, or by the pair of them (default),
|
|
7
|
+
switchable from the UI, the CLI (`--group-by`) and `Sidekiq::ErrorGroups.config`.
|
|
8
|
+
- Per-group retry/delete/kill actions, scoped to whichever half of the key is given.
|
|
9
|
+
- Ruby API, `rake sidekiq:error_groups` and the `sidekiq-error-groups` CLI.
|
data/LICENSE.txt
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
The MIT License (MIT)
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 r3cha
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
|
13
|
+
all copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
# sidekiq-error-groups
|
|
2
|
+
|
|
3
|
+
[](https://rubygems.org/gems/sidekiq-error-groups)
|
|
4
|
+
[](https://github.com/r3cha/sidekiq-error-groups/actions/workflows/ci.yml)
|
|
5
|
+
[](https://github.com/sidekiq/sidekiq)
|
|
6
|
+
[](LICENSE.txt)
|
|
7
|
+
|
|
8
|
+
An **Error groups** tab for the Sidekiq Web UI: retrying and dead jobs grouped by error
|
|
9
|
+
class, by job class, or by the pair of them — with retry, kill and delete scoped to one
|
|
10
|
+
group.
|
|
11
|
+
|
|
12
|
+

|
|
13
|
+
|
|
14
|
+
## Why
|
|
15
|
+
|
|
16
|
+
Sidekiq's **Retries** and **Dead** pages are flat lists sorted by time. With 20 000 failed
|
|
17
|
+
jobs they cannot answer the only question that matters — *what broke?* — and the bulk
|
|
18
|
+
actions are all-or-nothing.
|
|
19
|
+
|
|
20
|
+
And grouping by error class alone is not enough either. `Net::OpenTimeout` in
|
|
21
|
+
`FetchInventoryJob` is a flaky warehouse API; the same `Net::OpenTimeout` in
|
|
22
|
+
`DeliverWebhookJob` is a flaky customer endpoint. Two problems, two decisions. Pairing the
|
|
23
|
+
error class with the job class keeps them apart, so you can retry one and leave the other
|
|
24
|
+
alone. That is the default grouping; the other two are one click away.
|
|
25
|
+
|
|
26
|
+
Nothing is stored and nothing is instrumented: the tab reads the very same retry and dead
|
|
27
|
+
sets Sidekiq already keeps, so it works on failures that happened before you installed it.
|
|
28
|
+
|
|
29
|
+
## Installation
|
|
30
|
+
|
|
31
|
+
```ruby
|
|
32
|
+
# Gemfile
|
|
33
|
+
gem "sidekiq-error-groups"
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Require the Web extension where the Web UI is mounted — `config/routes.rb` in Rails, or
|
|
37
|
+
your rackup file. **Not** in an initializer: Sidekiq worker processes should not load the
|
|
38
|
+
Web UI.
|
|
39
|
+
|
|
40
|
+
```ruby
|
|
41
|
+
# config/routes.rb
|
|
42
|
+
require "sidekiq/web"
|
|
43
|
+
require "sidekiq/error_groups/web"
|
|
44
|
+
|
|
45
|
+
Rails.application.routes.draw do
|
|
46
|
+
mount Sidekiq::Web => "/sidekiq"
|
|
47
|
+
end
|
|
48
|
+
```
|
|
49
|
+
|
|
50
|
+
That is the whole setup. A new "Error groups" tab appears next to Retries and Dead.
|
|
51
|
+
|
|
52
|
+
## The tab
|
|
53
|
+
|
|
54
|
+
**Index** — one row per group, with a `Group by: error class · job · error class + job`
|
|
55
|
+
switch. Each row carries the totals per set, when the group first and last failed, and the
|
|
56
|
+
message of its most recent failure. When the grouping leaves something out, it comes back
|
|
57
|
+
as a breakdown column: job classes when grouping by error, error classes when grouping by
|
|
58
|
+
job.
|
|
59
|
+
|
|
60
|
+

|
|
61
|
+
|
|
62
|
+
**Group page** — the numbers for that group, the message and backtrace of the most recent
|
|
63
|
+
failure, the three bulk actions, and a paginated list of the jobs linking back to their
|
|
64
|
+
regular Sidekiq pages.
|
|
65
|
+
|
|
66
|
+

|
|
67
|
+
|
|
68
|
+
| Action | Effect |
|
|
69
|
+
|---|---|
|
|
70
|
+
| Retry all in group | pushes every job of this group back to its queue (retries **and** dead) |
|
|
71
|
+
| Kill all in group | moves this group's retrying jobs to the dead set |
|
|
72
|
+
| Delete all in group | deletes every job of this group (retries **and** dead) |
|
|
73
|
+
|
|
74
|
+
Each action asks for confirmation, stays inside the group it was fired from, and reports
|
|
75
|
+
how many jobs it touched.
|
|
76
|
+
|
|
77
|
+
## Ruby API
|
|
78
|
+
|
|
79
|
+
```ruby
|
|
80
|
+
summary = Sidekiq::ErrorGroups.summary # grouped by error class + job class
|
|
81
|
+
summary.total # => 72 failed jobs in the scanned sets
|
|
82
|
+
summary.group_by # => :error_job
|
|
83
|
+
summary.truncated? # => false (true when max_scan stopped the scan early)
|
|
84
|
+
|
|
85
|
+
summary.groups.each do |group|
|
|
86
|
+
group.error_class # => "Net::OpenTimeout"
|
|
87
|
+
group.job_class # => "FetchInventoryJob"
|
|
88
|
+
group.count # => 8
|
|
89
|
+
group.retry_count # => 8
|
|
90
|
+
group.dead_count # => 0
|
|
91
|
+
group.first_failed_at
|
|
92
|
+
group.last_failed_at
|
|
93
|
+
group.sample # => the group's most recent failed job
|
|
94
|
+
end
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
Other groupings, and the breakdown of whatever is not in the key:
|
|
98
|
+
|
|
99
|
+
```ruby
|
|
100
|
+
Sidekiq::ErrorGroups.summary(group_by: :error) # one row per error class
|
|
101
|
+
Sidekiq::ErrorGroups.summary(group_by: :job) # one row per job class
|
|
102
|
+
Sidekiq::ErrorGroups.summary(sets: [:dead]) # dead jobs only
|
|
103
|
+
|
|
104
|
+
group = Sidekiq::ErrorGroups.groups(group_by: :job).first
|
|
105
|
+
group.error_classes # => {"Net::OpenTimeout" => 5, "Net::ReadTimeout" => 5}
|
|
106
|
+
group.breakdown # => [["Net::OpenTimeout", 5], ["Net::ReadTimeout", 5]]
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
One group's jobs, paginated (`page.group` carries the whole-group numbers, not just the
|
|
110
|
+
page's):
|
|
111
|
+
|
|
112
|
+
```ruby
|
|
113
|
+
page = Sidekiq::ErrorGroups.jobs("Net::OpenTimeout", job_class: "FetchInventoryJob", page: 1)
|
|
114
|
+
page.jobs.first.jid
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
Bulk actions, scoped to whichever half of the key you pass:
|
|
118
|
+
|
|
119
|
+
```ruby
|
|
120
|
+
Sidekiq::ErrorGroups.retry_all("Net::OpenTimeout", job_class: "FetchInventoryJob")
|
|
121
|
+
Sidekiq::ErrorGroups.delete_all("ActiveRecord::RecordNotFound", sets: [:dead])
|
|
122
|
+
Sidekiq::ErrorGroups.kill_all(job_class: "WarmCacheJob", limit: 100)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
An action with neither half raises `ArgumentError` rather than quietly touching every
|
|
126
|
+
failed job. Jobs carrying no `error_class` are grouped under
|
|
127
|
+
`Sidekiq::ErrorGroups::Selector::UNKNOWN` and displayed as "(no error class)".
|
|
128
|
+
|
|
129
|
+
## Rake and CLI
|
|
130
|
+
|
|
131
|
+
```bash
|
|
132
|
+
bundle exec rake sidekiq:error_groups # GROUP_BY=error_job SETS=retry,dead LIMIT=20 MAX_SCAN=10000
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
```
|
|
136
|
+
$ sidekiq-error-groups --group-by job
|
|
137
|
+
Failed jobs grouped by job class — 72 job(s) (retry: 62, dead: 10), 7 group(s)
|
|
138
|
+
|
|
139
|
+
JOB TOTAL RETRY DEAD LAST FAILURE ERRORS
|
|
140
|
+
----------------- ----- ----- ---- ----------------------- ----------------------------------------------------------
|
|
141
|
+
ChargeCardJob 30 30 0 2026-10-06 08:48:05 UTC PaymentGateway::DeclinedError (26), …RateLimitedError (4)
|
|
142
|
+
DeliverWebhookJob 10 10 0 2026-10-06 08:46:12 UTC Net::OpenTimeout (5), Net::ReadTimeout (5)
|
|
143
|
+
SyncUserJob 9 9 0 2026-10-06 08:30:41 UTC ActiveRecord::RecordNotFound (9)
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
sidekiq-error-groups # the default table
|
|
148
|
+
sidekiq-error-groups --group-by error # or job, or error_job
|
|
149
|
+
sidekiq-error-groups --json | jq . # machine readable
|
|
150
|
+
sidekiq-error-groups --error "Net::OpenTimeout" # the jobs of one error class
|
|
151
|
+
sidekiq-error-groups --job FetchInventoryJob # the jobs of one worker
|
|
152
|
+
sidekiq-error-groups --error "Net::OpenTimeout" --job FetchInventoryJob
|
|
153
|
+
sidekiq-error-groups -r ./config/environment # load the app first
|
|
154
|
+
sidekiq-error-groups --redis-url redis://localhost:6379/1
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
The CLI is read-only on purpose: bulk retry/delete/kill live in the Ruby API and in the Web
|
|
158
|
+
UI, where there is a confirmation step.
|
|
159
|
+
|
|
160
|
+
## Configuration
|
|
161
|
+
|
|
162
|
+
Everything is optional.
|
|
163
|
+
|
|
164
|
+
```ruby
|
|
165
|
+
Sidekiq::ErrorGroups.configure do |config|
|
|
166
|
+
config.group_by = :error_job # :error, :job or :error_job (default)
|
|
167
|
+
config.sets = %i[retry dead] # which sets to scan, default both
|
|
168
|
+
config.max_scan = 10_000 # max jobs read per set, the safety valve for huge sets
|
|
169
|
+
config.action_batch_size = 1_000
|
|
170
|
+
config.top_breakdown = 3 # entries listed in the breakdown column
|
|
171
|
+
config.per_page = 25 # jobs per page on the group page
|
|
172
|
+
config.backtrace_lines = 10
|
|
173
|
+
config.cache_ttl = 0 # seconds to keep a summary in process memory, 0 = off
|
|
174
|
+
end
|
|
175
|
+
```
|
|
176
|
+
|
|
177
|
+
### Performance
|
|
178
|
+
|
|
179
|
+
Grouping means reading the sets, so the work is linear in their size. Iteration goes
|
|
180
|
+
through `Sidekiq::JobSet#each`, which paginates the sorted set inside Redis, and stops
|
|
181
|
+
after `max_scan` jobs per set. When that limit kicks in, the UI, the CLI and
|
|
182
|
+
`Summary#truncated?` all say so — the numbers are then a lower bound. Raise `max_scan` for
|
|
183
|
+
exact numbers on large installations, or set `cache_ttl` to avoid rescanning on every page
|
|
184
|
+
load.
|
|
185
|
+
|
|
186
|
+
Bulk actions deliberately ignore `max_scan`: a "retry all in group" must not silently skip
|
|
187
|
+
jobs. They work in batches of `action_batch_size`, re-scanning the set between batches,
|
|
188
|
+
because mutating a sorted set while iterating it is not safe.
|
|
189
|
+
|
|
190
|
+
## Compatibility
|
|
191
|
+
|
|
192
|
+
Ruby 2.7+ and Sidekiq 6.0 through 8.x. The Web UI extension API changed three times in that
|
|
193
|
+
range; the gem shims all of it and CI runs the full matrix.
|
|
194
|
+
|
|
195
|
+
| Sidekiq | Registration used | Views |
|
|
196
|
+
|---|---|---|
|
|
197
|
+
| 8.x | `Sidekiq::Web.configure { \|c\| c.register(...) }` | `web/views` (current UI markup) |
|
|
198
|
+
| 7.3–7.x | `Sidekiq::Web.register(ext, name:, tab:, index:)` | `web/views/legacy` |
|
|
199
|
+
| 6.0–7.2 | `Sidekiq::Web.register(ext)` + `Sidekiq::Web.tabs[...]` | `web/views/legacy` |
|
|
200
|
+
|
|
201
|
+
Only Sidekiq APIs present in every supported version are used (`Sidekiq::RetrySet`,
|
|
202
|
+
`Sidekiq::DeadSet`, `JobSet#each`, `SortedEntry#item/score/retry/delete/kill`), and Redis is
|
|
203
|
+
never addressed directly — which keeps both redis-rb (Sidekiq 6) and redis-client
|
|
204
|
+
(Sidekiq 7+) happy. Groups are addressed with `error_class`/`job_class` query parameters
|
|
205
|
+
rather than path segments, which every Sidekiq router version handles unchanged.
|
|
206
|
+
|
|
207
|
+
Locales: English and Russian.
|
|
208
|
+
|
|
209
|
+
## Development
|
|
210
|
+
|
|
211
|
+
```bash
|
|
212
|
+
bin/setup
|
|
213
|
+
bundle exec rake # tests + rubocop
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
The core is tested against in-memory fakes and needs nothing running. The Web UI tests
|
|
217
|
+
drive a real `Sidekiq::Web` through `rack-test` and need Redis; they skip themselves when
|
|
218
|
+
`REDIS_URL` (default `redis://localhost:6379/15`) is unreachable.
|
|
219
|
+
|
|
220
|
+
The whole version matrix:
|
|
221
|
+
|
|
222
|
+
```bash
|
|
223
|
+
for f in gemfiles/*.gemfile; do BUNDLE_GEMFILE=$f bundle install && BUNDLE_GEMFILE=$f bundle exec rake test; done
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
Looking at the UI by hand:
|
|
227
|
+
|
|
228
|
+
```bash
|
|
229
|
+
bundle exec ruby test/dummy/seed.rb # create failed jobs
|
|
230
|
+
bundle exec rackup test/dummy/config.ru -p 9292 # current UI
|
|
231
|
+
BUNDLE_GEMFILE=gemfiles/sidekiq_6.5.gemfile bundle exec rackup test/dummy/config.ru -p 9293 # legacy UI
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Bug reports and pull requests are welcome — see [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
235
|
+
|
|
236
|
+
## License
|
|
237
|
+
|
|
238
|
+
[MIT](LICENSE.txt)
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ErrorGroups
|
|
5
|
+
# Bulk retry/delete/kill restricted to a single group: one error class, one
|
|
6
|
+
# job class, or the pair of them.
|
|
7
|
+
#
|
|
8
|
+
# Mutating a sorted set while `Sidekiq::JobSet#each` walks it is not safe, so
|
|
9
|
+
# every pass works in two phases: collect a batch of matching jobs, then act
|
|
10
|
+
# on it. Each action removes the job from the set, so repeated passes make
|
|
11
|
+
# progress until nothing matches any more.
|
|
12
|
+
module Actions
|
|
13
|
+
OPERATIONS = {
|
|
14
|
+
retry: :retry!,
|
|
15
|
+
delete: :delete!,
|
|
16
|
+
kill: :kill!
|
|
17
|
+
}.freeze
|
|
18
|
+
|
|
19
|
+
# Killing means "move to the dead set", which only makes sense for jobs
|
|
20
|
+
# that are still being retried.
|
|
21
|
+
SUPPORTED_SETS = {
|
|
22
|
+
retry: %i[retry dead],
|
|
23
|
+
delete: %i[retry dead],
|
|
24
|
+
kill: %i[retry]
|
|
25
|
+
}.freeze
|
|
26
|
+
|
|
27
|
+
Result = Struct.new(:operation, :selector, :counts, keyword_init: true) do
|
|
28
|
+
def total
|
|
29
|
+
counts.values.sum
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def error_class
|
|
33
|
+
selector.error_class
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def job_class
|
|
37
|
+
selector.job_class
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def to_h
|
|
41
|
+
{operation: operation, **selector.to_h, counts: counts, total: total}
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
module_function
|
|
46
|
+
|
|
47
|
+
def call(error_class = nil, operation:, job_class: nil, sets: nil, limit: nil, sources: nil)
|
|
48
|
+
operation = operation.to_sym
|
|
49
|
+
method = OPERATIONS.fetch(operation) do
|
|
50
|
+
raise ArgumentError, "unknown operation: #{operation}, expected one of #{OPERATIONS.keys.join(", ")}"
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
selector = Selector.build(error_class: error_class, job_class: job_class)
|
|
54
|
+
requested = Config.normalize_sets(sets || ErrorGroups.config.sets)
|
|
55
|
+
counts = {}
|
|
56
|
+
|
|
57
|
+
(requested & SUPPORTED_SETS.fetch(operation)).each do |kind|
|
|
58
|
+
counts[kind] = process(kind, selector, method, limit, sources)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
ErrorGroups.reset_cache!
|
|
62
|
+
Result.new(operation: operation, selector: selector, counts: counts)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def retry_all(error_class = nil, **kwargs)
|
|
66
|
+
call(error_class, operation: :retry, **kwargs)
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
def delete_all(error_class = nil, **kwargs)
|
|
70
|
+
call(error_class, operation: :delete, **kwargs)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def kill_all(error_class = nil, **kwargs)
|
|
74
|
+
call(error_class, operation: :kill, **kwargs)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# @api private
|
|
78
|
+
def process(kind, selector, method, limit, sources)
|
|
79
|
+
set = Scanner.source_for(kind, sources)
|
|
80
|
+
batch_size = ErrorGroups.config.action_batch_size
|
|
81
|
+
done = 0
|
|
82
|
+
|
|
83
|
+
loop do
|
|
84
|
+
take = batch_size
|
|
85
|
+
if limit
|
|
86
|
+
remaining = limit - done
|
|
87
|
+
break if remaining <= 0
|
|
88
|
+
|
|
89
|
+
take = [batch_size, remaining].min
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
batch = collect(set, kind, selector, take)
|
|
93
|
+
break if batch.empty?
|
|
94
|
+
|
|
95
|
+
size_before = Scanner.size_of(set)
|
|
96
|
+
batch.each { |job| job.public_send(method) }
|
|
97
|
+
done += batch.size
|
|
98
|
+
|
|
99
|
+
break if batch.size < take
|
|
100
|
+
# The set did not shrink, so another pass would collect the very same
|
|
101
|
+
# jobs again. Stop instead of spinning forever.
|
|
102
|
+
break if Scanner.size_of(set) == size_before
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
done
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
# @api private
|
|
109
|
+
def collect(set, kind, selector, take)
|
|
110
|
+
batch = []
|
|
111
|
+
|
|
112
|
+
set.each do |entry|
|
|
113
|
+
job = Job.new(kind, entry)
|
|
114
|
+
next unless selector.matches?(job)
|
|
115
|
+
|
|
116
|
+
batch << job
|
|
117
|
+
break if batch.size >= take
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
batch
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ErrorGroups
|
|
5
|
+
# Builds a {Sidekiq::ErrorGroups::Summary} out of the retry/dead sets.
|
|
6
|
+
class Aggregator
|
|
7
|
+
def self.call(**kwargs)
|
|
8
|
+
new(**kwargs).call
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def initialize(sets: nil, group_by: nil, max_scan: nil, sources: nil)
|
|
12
|
+
@sets = Config.normalize_sets(sets || ErrorGroups.config.sets)
|
|
13
|
+
@group_by = Config.normalize_group_by(group_by || ErrorGroups.config.group_by)
|
|
14
|
+
@max_scan = max_scan.nil? ? ErrorGroups.config.max_scan : max_scan
|
|
15
|
+
@sources = sources
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
def call
|
|
19
|
+
groups = {}
|
|
20
|
+
|
|
21
|
+
scanned, sizes = Scanner.scan(sets: @sets, max_scan: @max_scan, sources: @sources) do |job|
|
|
22
|
+
selector = Selector.for(job, @group_by)
|
|
23
|
+
(groups[selector.key] ||= Group.new(selector)).add(job)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
Summary.new(groups: sort(groups.values), group_by: @group_by, scanned: scanned, set_sizes: sizes)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
# Biggest problem first; ties broken by recency, then alphabetically so the
|
|
32
|
+
# output is stable between requests.
|
|
33
|
+
def sort(groups)
|
|
34
|
+
groups.sort_by do |group|
|
|
35
|
+
[-group.count, -(group.last_failed_at&.to_f || 0), group.selector.to_s]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "optparse"
|
|
4
|
+
require "json"
|
|
5
|
+
require_relative "../error_groups"
|
|
6
|
+
|
|
7
|
+
module Sidekiq
|
|
8
|
+
module ErrorGroups
|
|
9
|
+
# `sidekiq-error-groups` executable: prints the same summary the Web UI tab
|
|
10
|
+
# shows. Read-only on purpose — bulk retry/delete/kill live in the Ruby API
|
|
11
|
+
# and in the Web UI, where there is a confirmation step.
|
|
12
|
+
class CLI
|
|
13
|
+
BANNER = "Usage: sidekiq-error-groups [options]"
|
|
14
|
+
|
|
15
|
+
def self.run(argv, out: $stdout, err: $stderr)
|
|
16
|
+
new(argv).run(out: out, err: err)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def initialize(argv)
|
|
20
|
+
@argv = argv
|
|
21
|
+
@options = {
|
|
22
|
+
sets: nil, group_by: nil, limit: nil, max_scan: nil, json: false,
|
|
23
|
+
requires: [], error_class: nil, job_class: nil, show: false, page: 1
|
|
24
|
+
}
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def run(out: $stdout, err: $stderr)
|
|
28
|
+
parse!
|
|
29
|
+
boot!
|
|
30
|
+
out.puts(@options[:show] ? render_group : render_summary)
|
|
31
|
+
0
|
|
32
|
+
rescue OptionParser::ParseError, ArgumentError => e
|
|
33
|
+
err.puts("sidekiq-error-groups: #{e.message}")
|
|
34
|
+
1
|
|
35
|
+
rescue StandardError => e
|
|
36
|
+
err.puts("sidekiq-error-groups: #{e.class}: #{e.message}")
|
|
37
|
+
1
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def parser
|
|
43
|
+
@parser ||= OptionParser.new do |opts|
|
|
44
|
+
opts.banner = BANNER
|
|
45
|
+
|
|
46
|
+
opts.on("-u", "--redis-url URL", "Redis URL (defaults to REDIS_URL)") { |v| @options[:redis_url] = v }
|
|
47
|
+
opts.on("-s", "--sets LIST", "Job sets to scan, default retry,dead") { |v| @options[:sets] = v.split(",") }
|
|
48
|
+
opts.on("-b", "--group-by MODE", "error, job or error_job (default #{ErrorGroups.config.group_by})") do |v|
|
|
49
|
+
@options[:group_by] = v
|
|
50
|
+
end
|
|
51
|
+
opts.on("-l", "--limit N", Integer, "Show only the N largest groups") { |v| @options[:limit] = v }
|
|
52
|
+
opts.on("-m", "--max-scan N", Integer,
|
|
53
|
+
"Max jobs to read per set, default #{ErrorGroups.config.max_scan}") do |v|
|
|
54
|
+
@options[:max_scan] = v
|
|
55
|
+
end
|
|
56
|
+
opts.on("-e", "--error ERROR_CLASS", "List the jobs of one error class") do |v|
|
|
57
|
+
@options[:error_class] = v
|
|
58
|
+
@options[:show] = true
|
|
59
|
+
end
|
|
60
|
+
opts.on("-g", "--job JOB_CLASS", "List the jobs of one job class") do |v|
|
|
61
|
+
@options[:job_class] = v
|
|
62
|
+
@options[:show] = true
|
|
63
|
+
end
|
|
64
|
+
opts.on("-p", "--page N", Integer, "Page for --error/--job, default 1") { |v| @options[:page] = v }
|
|
65
|
+
opts.on("-j", "--json", "Print JSON instead of a table") { @options[:json] = true }
|
|
66
|
+
opts.on("-r", "--require PATH", "Require a file first, e.g. ./config/environment") do |v|
|
|
67
|
+
@options[:requires] << v
|
|
68
|
+
end
|
|
69
|
+
opts.on("-V", "--version", "Print version and exit") do
|
|
70
|
+
puts "sidekiq-error-groups #{VERSION} (sidekiq #{Sidekiq::VERSION})"
|
|
71
|
+
exit 0
|
|
72
|
+
end
|
|
73
|
+
opts.on("-h", "--help", "Print this help") do
|
|
74
|
+
puts opts
|
|
75
|
+
exit 0
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def parse!
|
|
81
|
+
parser.parse!(@argv)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def boot!
|
|
85
|
+
@options[:requires].each { |path| require File.expand_path(path) }
|
|
86
|
+
|
|
87
|
+
url = @options[:redis_url]
|
|
88
|
+
Sidekiq.configure_client do |config|
|
|
89
|
+
config.redis = {url: url} if url
|
|
90
|
+
# Keep stdout clean: the report goes there, Sidekiq's own chatter does not.
|
|
91
|
+
config.logger = Sidekiq::Logger.new($stderr, level: :warn)
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def render_summary
|
|
96
|
+
summary = ErrorGroups.summary(
|
|
97
|
+
sets: @options[:sets], group_by: @options[:group_by], max_scan: @options[:max_scan], refresh: true
|
|
98
|
+
)
|
|
99
|
+
return JSON.pretty_generate(summary.to_h) if @options[:json]
|
|
100
|
+
|
|
101
|
+
Formatter.render(summary, limit: @options[:limit])
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def render_group
|
|
105
|
+
error_class = @options[:error_class]
|
|
106
|
+
error_class = Selector::UNKNOWN if error_class == Formatter::NO_ERROR_CLASS
|
|
107
|
+
|
|
108
|
+
page = ErrorGroups.jobs(
|
|
109
|
+
error_class,
|
|
110
|
+
job_class: @options[:job_class],
|
|
111
|
+
sets: @options[:sets],
|
|
112
|
+
max_scan: @options[:max_scan],
|
|
113
|
+
page: @options[:page],
|
|
114
|
+
per_page: @options[:limit]
|
|
115
|
+
)
|
|
116
|
+
return JSON.pretty_generate(page_to_h(page)) if @options[:json]
|
|
117
|
+
|
|
118
|
+
format_group(page)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def page_to_h(page)
|
|
122
|
+
{
|
|
123
|
+
**page.selector.to_h,
|
|
124
|
+
total: page.total,
|
|
125
|
+
page: page.page,
|
|
126
|
+
pages: page.pages,
|
|
127
|
+
truncated: page.truncated,
|
|
128
|
+
jobs: page.jobs.map(&:to_h)
|
|
129
|
+
}
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def format_group(page)
|
|
133
|
+
lines = ["#{selector_label(page.selector)} — #{page.total} job(s), page #{page.page}/#{page.pages}"]
|
|
134
|
+
lines << "Only the first #{page.scanned} job(s) were scanned (max_scan)." if page.truncated
|
|
135
|
+
lines << ""
|
|
136
|
+
|
|
137
|
+
page.jobs.each do |job|
|
|
138
|
+
lines << "#{Formatter.timestamp(job.last_failure_at)} [#{job.set}] #{job.display_class} " \
|
|
139
|
+
"#{Formatter.error_class_label(job.error_class)} " \
|
|
140
|
+
"jid=#{job.jid} queue=#{job.queue} retries=#{job.retry_count}"
|
|
141
|
+
lines << " #{job.error_message}" unless job.error_message.empty?
|
|
142
|
+
end
|
|
143
|
+
lines << "(no jobs on this page)" if page.jobs.empty?
|
|
144
|
+
lines.join("\n")
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def selector_label(selector)
|
|
148
|
+
[
|
|
149
|
+
selector.error_class && Formatter.error_class_label(selector.error_class),
|
|
150
|
+
selector.job_class
|
|
151
|
+
].compact.join(" · ")
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Sidekiq
|
|
4
|
+
module ErrorGroups
|
|
5
|
+
# Sidekiq changed its Web UI extension API several times between 6.0 and 8.x.
|
|
6
|
+
# Every version check in this gem goes through here, so the differences stay
|
|
7
|
+
# in one place.
|
|
8
|
+
module Compat
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
def sidekiq_version
|
|
12
|
+
@sidekiq_version ||= Gem::Version.new(Sidekiq::VERSION)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def at_least?(version)
|
|
16
|
+
sidekiq_version >= Gem::Version.new(version)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Web UI rewritten: `Sidekiq::Web.configure`, `route_params(:key)`,
|
|
20
|
+
# `url_params("key")` and a brand new markup/CSS.
|
|
21
|
+
def sidekiq_8?
|
|
22
|
+
at_least?("8.0.0")
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# `Sidekiq::Web.register` gained the `name:`/`tab:`/`index:` keywords.
|
|
26
|
+
def sidekiq_73?
|
|
27
|
+
at_least?("7.3.0")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Reset memoized state, used by the test suite.
|
|
31
|
+
def reset!
|
|
32
|
+
@sidekiq_version = nil
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|