roundhouse_ui 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +31 -0
- data/app/controllers/concerns/roundhouse_ui/job_set_browsing.rb +20 -0
- data/app/controllers/roundhouse_ui/dashboard_controller.rb +5 -0
- data/app/controllers/roundhouse_ui/dead_controller.rb +12 -1
- data/app/controllers/roundhouse_ui/errors_controller.rb +3 -51
- data/app/controllers/roundhouse_ui/metrics_controller.rb +1 -0
- data/app/controllers/roundhouse_ui/retries_controller.rb +11 -1
- data/app/views/layouts/roundhouse_ui/application.html.erb +97 -7
- data/app/views/roundhouse_ui/dashboard/show.html.erb +56 -22
- data/app/views/roundhouse_ui/dead/index.html.erb +8 -0
- data/app/views/roundhouse_ui/jobs/show.html.erb +13 -2
- data/app/views/roundhouse_ui/metrics/show.html.erb +21 -0
- data/app/views/roundhouse_ui/retries/index.html.erb +9 -0
- data/config/routes.rb +5 -3
- data/lib/roundhouse_ui/duration_collector.rb +53 -0
- data/lib/roundhouse_ui/error_groups.rb +65 -0
- data/lib/roundhouse_ui/health.rb +65 -0
- data/lib/roundhouse_ui/version.rb +1 -1
- data/lib/roundhouse_ui.rb +10 -0
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 11537f41a49c350dbd057961b82994fdc7524afdf3285961a28fa13701daaf57
|
|
4
|
+
data.tar.gz: aedef27c3202b3d2eaf25a2b9fefb9290df222c66d1fde4ae8414152f3a477cf
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: f4b22e1a716bbbdfdbab62cc5b0dcb10094a3cefbefd7ad1f1a24367e1f383f4c4c1467d1aa840600f860b1bc65b433fbdbdc56da5e1f1c1545f1756a99ed83c
|
|
7
|
+
data.tar.gz: cbe28c835c0e6413ee88658242e455e39a47cbd124a1e06ee0a9d40106ea9eb3ba49e2491f8dbd6f3a757ad71c0a24f5ead03b0b263b8a96237db3d2f8732b74
|
data/README.md
CHANGED
|
@@ -88,6 +88,14 @@ RoundhouseUi.configure do |c|
|
|
|
88
88
|
# e.g. when you run reliable fetch (super_fetch) instead of RoundhouseUi::Fetch.
|
|
89
89
|
# Default: true.
|
|
90
90
|
# c.pause_enabled = false
|
|
91
|
+
|
|
92
|
+
# Seconds between dashboard stat polls (default 5). Raise it if polling shows
|
|
93
|
+
# up in your traces — each poll re-runs the host's auth/routing on the mount.
|
|
94
|
+
# c.poll_interval = 10
|
|
95
|
+
|
|
96
|
+
# Show the "slowest job classes" table on the Metrics page. Requires the
|
|
97
|
+
# DurationCollector middleware (see below). Default: false.
|
|
98
|
+
# c.collect_durations = true
|
|
91
99
|
end
|
|
92
100
|
```
|
|
93
101
|
|
|
@@ -150,6 +158,29 @@ The **Busy** page's Cancel button flags a job's JID. A queued/scheduled/retrying
|
|
|
150
158
|
is then skipped when it would next run; a *currently running* job stops only if it
|
|
151
159
|
checks in — e.g. a long loop can `break if RoundhouseUi.cancelled?(jid)`.
|
|
152
160
|
|
|
161
|
+
## Slowest job classes
|
|
162
|
+
|
|
163
|
+
Sidekiq doesn't track per-class durations, so Roundhouse can record them itself.
|
|
164
|
+
Install the opt-in server middleware and set `collect_durations = true`; the
|
|
165
|
+
Metrics page then lists the slowest classes by total time (count + average).
|
|
166
|
+
|
|
167
|
+
```ruby
|
|
168
|
+
# config/initializers/sidekiq.rb
|
|
169
|
+
Sidekiq.configure_server do |config|
|
|
170
|
+
config.server_middleware { |chain| chain.add RoundhouseUi::DurationCollector }
|
|
171
|
+
end
|
|
172
|
+
```
|
|
173
|
+
|
|
174
|
+
It's two cheap Redis writes per job (a counter + a summed-ms float) into a single
|
|
175
|
+
hash, and a job failure never propagates from the collector.
|
|
176
|
+
|
|
177
|
+
## Bulk actions on a filter
|
|
178
|
+
|
|
179
|
+
On **Retries** and **Dead**, searching narrows the set; with a filter active you
|
|
180
|
+
can retry or delete **every** matching job in one action (not just the visible
|
|
181
|
+
page), capped at 1,000 per run. Gated to when a filter is present so it can't
|
|
182
|
+
become "retry everything", `read_only`-aware, and audit-logged.
|
|
183
|
+
|
|
153
184
|
## Observability deep-links
|
|
154
185
|
|
|
155
186
|
The core depends on nothing — it asks the configured adapter for a URL and renders a link
|
|
@@ -5,6 +5,7 @@ module RoundhouseUi
|
|
|
5
5
|
extend ActiveSupport::Concern
|
|
6
6
|
|
|
7
7
|
PER_PAGE = 25
|
|
8
|
+
BULK_CAP = 1_000 # safety ceiling on a single match-set action
|
|
8
9
|
|
|
9
10
|
# Returns [entries_for_page, has_next?]. Scans only far enough to fill the
|
|
10
11
|
# requested page plus one (to know if a next page exists) — never loads the
|
|
@@ -32,6 +33,25 @@ module RoundhouseUi
|
|
|
32
33
|
[ jobs, has_next ]
|
|
33
34
|
end
|
|
34
35
|
|
|
36
|
+
# Apply an op ("retry"/"delete") to every entry matching the query, capped at
|
|
37
|
+
# BULK_CAP. Entries are collected first, then acted on — mutating a Sidekiq set
|
|
38
|
+
# mid-iteration skips entries. Returns [count_acted_on, capped?].
|
|
39
|
+
def bulk_apply(set, query, op, cap = BULK_CAP)
|
|
40
|
+
matches = []
|
|
41
|
+
capped = false
|
|
42
|
+
set.each do |entry|
|
|
43
|
+
next if query.present? && !entry_matches?(entry, query)
|
|
44
|
+
|
|
45
|
+
matches << entry
|
|
46
|
+
if matches.size >= cap
|
|
47
|
+
capped = true
|
|
48
|
+
break
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
matches.each { |entry| op == "delete" ? entry.delete : entry.retry }
|
|
52
|
+
[ matches.size, capped ]
|
|
53
|
+
end
|
|
54
|
+
|
|
35
55
|
def entry_matches?(entry, query)
|
|
36
56
|
needle = query.downcase
|
|
37
57
|
[ entry.klass, entry.jid, entry.item["error_class"], entry.item["error_message"], entry.args.to_s ]
|
|
@@ -5,6 +5,11 @@ module RoundhouseUi
|
|
|
5
5
|
def show
|
|
6
6
|
@stats = Sidekiq::Stats.new
|
|
7
7
|
@queues = Sidekiq::Queue.all
|
|
8
|
+
@metrics = Metrics.new(stats: @stats)
|
|
9
|
+
@health = Health.new(stats: @stats, queues: @queues, metrics: @metrics)
|
|
10
|
+
# Highest-signal slices for the overview, from data we already read.
|
|
11
|
+
@top_errors = ErrorGroups.new(limit: 200).call.groups.first(5)
|
|
12
|
+
@problem_queues = @queues.select { |q| q.latency > 5 }.sort_by { |q| -q.latency }.first(5)
|
|
8
13
|
end
|
|
9
14
|
|
|
10
15
|
# Polled by the dashboard for live counts (same approach Sidekiq Web uses —
|
|
@@ -2,7 +2,7 @@ module RoundhouseUi
|
|
|
2
2
|
class DeadController < ApplicationController
|
|
3
3
|
include JobSetBrowsing
|
|
4
4
|
|
|
5
|
-
before_action :require_writable!, only: %i[requeue destroy bulk]
|
|
5
|
+
before_action :require_writable!, only: %i[requeue destroy bulk bulk_all]
|
|
6
6
|
|
|
7
7
|
def index
|
|
8
8
|
@query = params[:q].to_s.strip
|
|
@@ -36,6 +36,17 @@ module RoundhouseUi
|
|
|
36
36
|
redirect_to dead_set_path, notice: "#{verb} #{count} job(s)."
|
|
37
37
|
end
|
|
38
38
|
|
|
39
|
+
# Smart bulk: act on EVERY job matching the current filter (not just the
|
|
40
|
+
# selected/visible ones), capped for safety. Only offered when a filter is
|
|
41
|
+
# active, so it can't become "retry the entire dead set" by accident.
|
|
42
|
+
def bulk_all
|
|
43
|
+
count, capped = bulk_apply(Sidekiq::DeadSet.new, params[:q].to_s.strip, params[:op])
|
|
44
|
+
verb = params[:op] == "delete" ? "Deleted" : "Re-enqueued"
|
|
45
|
+
note = "#{verb} #{count} matching job(s)."
|
|
46
|
+
note += " Stopped at the #{JobSetBrowsing::BULK_CAP} cap — run again for more." if capped
|
|
47
|
+
redirect_to dead_set_path, notice: note
|
|
48
|
+
end
|
|
49
|
+
|
|
39
50
|
private
|
|
40
51
|
|
|
41
52
|
def require_writable!
|
|
@@ -3,59 +3,11 @@ module RoundhouseUi
|
|
|
3
3
|
# (job class + error class) — so one bad deploy reads as a single issue with
|
|
4
4
|
# a count, not five thousand identical rows. The aggregation Sidekiq Web lacks.
|
|
5
5
|
class ErrorsController < ApplicationController
|
|
6
|
-
SCAN_LIMIT = 1_000 # cap entries scanned per pass; shown honestly in the view
|
|
7
|
-
|
|
8
6
|
def index
|
|
9
7
|
@query = params[:q].to_s.strip
|
|
10
|
-
@scan_limit =
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
private
|
|
15
|
-
|
|
16
|
-
def aggregate
|
|
17
|
-
groups = {}
|
|
18
|
-
scanned = 0
|
|
19
|
-
truncated = false
|
|
20
|
-
|
|
21
|
-
sources.each do |source, set|
|
|
22
|
-
set.each do |entry|
|
|
23
|
-
scanned += 1
|
|
24
|
-
if scanned > SCAN_LIMIT
|
|
25
|
-
truncated = true
|
|
26
|
-
break
|
|
27
|
-
end
|
|
28
|
-
record(groups, source, entry)
|
|
29
|
-
end
|
|
30
|
-
break if truncated
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
list = groups.values.sort_by { |g| -g[:count] }
|
|
34
|
-
list = list.select { |g| "#{g[:klass]} #{g[:error]}".downcase.include?(@query.downcase) } if @query.present?
|
|
35
|
-
[ list, scanned, truncated ]
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Sidekiq's native sets, plus the sidekiq-failures `failed` set when opted in
|
|
39
|
-
# and that gem is loaded. Its FailureSet is a Sidekiq::JobSet, so it iterates
|
|
40
|
-
# exactly like the others — no special-casing in the aggregation above.
|
|
41
|
-
def sources
|
|
42
|
-
sets = { "retry" => Sidekiq::RetrySet.new, "dead" => Sidekiq::DeadSet.new }
|
|
43
|
-
if RoundhouseUi.show_sidekiq_failures && defined?(Sidekiq::Failures::FailureSet)
|
|
44
|
-
sets["failed"] = Sidekiq::Failures::FailureSet.new
|
|
45
|
-
end
|
|
46
|
-
sets
|
|
47
|
-
end
|
|
48
|
-
|
|
49
|
-
def record(groups, source, entry)
|
|
50
|
-
error = entry.item["error_class"] || "UnknownError"
|
|
51
|
-
group = (groups["#{entry.klass}|#{error}"] ||= {
|
|
52
|
-
klass: entry.klass, error: error, count: 0, last_at: nil, queues: [], sources: []
|
|
53
|
-
})
|
|
54
|
-
group[:count] += 1
|
|
55
|
-
group[:queues] |= [ entry.queue ]
|
|
56
|
-
group[:sources] |= [ source ]
|
|
57
|
-
at = entry.at
|
|
58
|
-
group[:last_at] = at if at && (group[:last_at].nil? || at > group[:last_at])
|
|
8
|
+
@scan_limit = ErrorGroups::DEFAULT_SCAN_LIMIT
|
|
9
|
+
result = ErrorGroups.new(query: @query).call
|
|
10
|
+
@groups, @scanned, @truncated = result.groups, result.scanned, result.truncated
|
|
59
11
|
end
|
|
60
12
|
end
|
|
61
13
|
end
|
|
@@ -2,7 +2,7 @@ module RoundhouseUi
|
|
|
2
2
|
class RetriesController < ApplicationController
|
|
3
3
|
include JobSetBrowsing
|
|
4
4
|
|
|
5
|
-
before_action :require_writable!, only: %i[requeue destroy]
|
|
5
|
+
before_action :require_writable!, only: %i[requeue destroy bulk_all]
|
|
6
6
|
|
|
7
7
|
def index
|
|
8
8
|
@query = params[:q].to_s.strip
|
|
@@ -24,6 +24,16 @@ module RoundhouseUi
|
|
|
24
24
|
redirect_to retries_path, notice: entry ? "Deleted #{params[:jid]}." : "Job is no longer in the retry set."
|
|
25
25
|
end
|
|
26
26
|
|
|
27
|
+
# Smart bulk: retry/delete EVERY job matching the current filter, capped for
|
|
28
|
+
# safety. Offered only when a filter is active.
|
|
29
|
+
def bulk_all
|
|
30
|
+
count, capped = bulk_apply(Sidekiq::RetrySet.new, params[:q].to_s.strip, params[:op])
|
|
31
|
+
verb = params[:op] == "delete" ? "Deleted" : "Re-enqueued"
|
|
32
|
+
note = "#{verb} #{count} matching job(s)."
|
|
33
|
+
note += " Stopped at the #{JobSetBrowsing::BULK_CAP} cap — run again for more." if capped
|
|
34
|
+
redirect_to retries_path, notice: note
|
|
35
|
+
end
|
|
36
|
+
|
|
27
37
|
private
|
|
28
38
|
|
|
29
39
|
def require_writable!
|
|
@@ -117,12 +117,73 @@
|
|
|
117
117
|
.rh-card .d.up { color:var(--good); } .rh-card .d.bad { color:var(--crit); }
|
|
118
118
|
.rh-card .pill { display:inline-flex; align-items:center; gap:6px; font-size:13px; font-weight:600; padding:4px 10px; border-radius:7px; margin-bottom:9px; background:rgba(226,165,63,.14); color:var(--warn); }
|
|
119
119
|
.rh-card .pill.ok { background:rgba(68,197,140,.14); color:var(--good); }
|
|
120
|
-
.rh-chart-wrap { background:var(--panel); border:1px solid var(--line-soft); border-radius:12px; padding:18px 20px 10px; margin-bottom:24px; }
|
|
120
|
+
.rh-chart-wrap { background:var(--panel); border:1px solid var(--line-soft); border-radius:12px; padding:18px 20px 10px; margin-bottom:24px; position:relative; }
|
|
121
|
+
#rh-chart-tip { position:absolute; top:34px; transform:translateX(-50%); pointer-events:none; background:var(--panel-3); border:1px solid var(--line); color:var(--text); font:11px var(--mono); padding:2px 7px; border-radius:6px; white-space:nowrap; }
|
|
122
|
+
#rh-chart-tip[hidden] { display:none; }
|
|
121
123
|
.rh-chart-wrap .top { display:flex; align-items:baseline; gap:12px; margin-bottom:6px; }
|
|
122
124
|
.rh-chart-wrap h3 { font-size:13px; font-weight:600; margin:0; }
|
|
123
125
|
.rh-chart-wrap .now { margin-left:auto; font-family:var(--mono); font-size:13px; color:var(--accent); }
|
|
124
126
|
canvas#rh-chart { display:block; width:100%; height:90px; }
|
|
125
127
|
.rh-cb { width:15px; height:15px; }
|
|
128
|
+
|
|
129
|
+
/* composite health banner */
|
|
130
|
+
.rh-health { background:var(--panel); border:1px solid var(--line-soft); border-left:3px solid var(--faint); border-radius:12px; margin-bottom:18px; }
|
|
131
|
+
.rh-health-ok { border-left-color:var(--good); }
|
|
132
|
+
.rh-health-warn { border-left-color:var(--warn); }
|
|
133
|
+
.rh-health-crit { border-left-color:var(--crit); }
|
|
134
|
+
.rh-health summary { display:flex; align-items:center; gap:12px; padding:14px 16px; cursor:pointer; list-style:none; }
|
|
135
|
+
.rh-health summary::-webkit-details-marker { display:none; }
|
|
136
|
+
.rh-health-dot { width:9px; height:9px; border-radius:50%; flex:none; background:var(--faint); }
|
|
137
|
+
.rh-health-ok .rh-health-dot { background:var(--good); }
|
|
138
|
+
.rh-health-warn .rh-health-dot { background:var(--warn); }
|
|
139
|
+
.rh-health-crit .rh-health-dot { background:var(--crit); }
|
|
140
|
+
.rh-health-verdict { font-weight:650; font-size:15px; }
|
|
141
|
+
.rh-health-ok .rh-health-verdict { color:var(--good); }
|
|
142
|
+
.rh-health-warn .rh-health-verdict { color:var(--warn); }
|
|
143
|
+
.rh-health-crit .rh-health-verdict { color:var(--crit); }
|
|
144
|
+
.rh-health-reason { color:var(--muted); font-size:13px; }
|
|
145
|
+
.rh-health-cta { margin-left:auto; font:11px var(--mono); color:var(--faint); }
|
|
146
|
+
.rh-health[open] .rh-health-cta { color:var(--accent); }
|
|
147
|
+
.rh-health-signals { padding:2px 16px 14px; display:flex; flex-direction:column; gap:8px; border-top:1px solid var(--line-soft); margin-top:2px; padding-top:12px; }
|
|
148
|
+
.rh-sig { display:flex; align-items:center; gap:10px; font-size:13px; }
|
|
149
|
+
.rh-sig b { min-width:170px; font-weight:600; }
|
|
150
|
+
|
|
151
|
+
/* insight panels */
|
|
152
|
+
.rh-insights { display:grid; gap:14px; margin-bottom:24px; }
|
|
153
|
+
@media(min-width:760px){ .rh-insights { grid-template-columns:1fr 1fr; } }
|
|
154
|
+
.rh-insight { padding:6px 0; }
|
|
155
|
+
.rh-insight-h { display:flex; align-items:baseline; justify-content:space-between; gap:10px; font-size:13px; font-weight:600; padding:13px 16px 10px; }
|
|
156
|
+
.rh-insight-row { display:flex; align-items:center; gap:12px; padding:10px 16px; border-top:1px solid var(--line-soft); }
|
|
157
|
+
.rh-insight-row .rh-insight-main { flex:1; min-width:0; font-size:13px; }
|
|
158
|
+
.rh-insight-row .rh-pill { font-variant-numeric:tabular-nums; }
|
|
159
|
+
.rh-insight .rh-empty { padding:18px 16px; }
|
|
160
|
+
|
|
161
|
+
/* collapsible disclosure (large args, full backtrace) */
|
|
162
|
+
.rh-disclose summary { cursor:pointer; font:12px var(--mono); color:var(--accent); padding:6px 0; list-style:none; }
|
|
163
|
+
.rh-disclose summary::-webkit-details-marker { display:none; }
|
|
164
|
+
.rh-disclose summary::before { content:"▸ "; }
|
|
165
|
+
.rh-disclose[open] summary::before { content:"▾ "; }
|
|
166
|
+
|
|
167
|
+
/* connection state — poll failures surface here instead of silently going stale */
|
|
168
|
+
.rh-live.is-stale .rh-dot { background:var(--crit); }
|
|
169
|
+
.rh-live.is-stale .rh-dot::after { animation:none; }
|
|
170
|
+
|
|
171
|
+
/* keyboard focus — visible on every interactive control */
|
|
172
|
+
a:focus-visible, button:focus-visible, input:focus-visible, select:focus-visible,
|
|
173
|
+
summary:focus-visible, .rh-nav:focus-visible { outline:2px solid var(--accent); outline-offset:2px; border-radius:6px; }
|
|
174
|
+
|
|
175
|
+
/* on-call / mobile: stack the rail on top as a horizontal nav, collapse the grid */
|
|
176
|
+
@media (max-width:760px) {
|
|
177
|
+
.rh-app { grid-template-columns:1fr; }
|
|
178
|
+
.rh-rail { position:static; height:auto; flex-direction:row; flex-wrap:wrap; gap:6px 14px; align-items:center; border-right:none; border-bottom:1px solid var(--line-soft); }
|
|
179
|
+
.rh-rail .rh-grp { flex-direction:row; flex-wrap:wrap; gap:4px; }
|
|
180
|
+
.rh-rail .rh-grp-label { display:none; }
|
|
181
|
+
.rh-brand { width:100%; }
|
|
182
|
+
.rh-main { padding:16px 16px 60px; }
|
|
183
|
+
.rh-cards { grid-template-columns:1fr 1fr; }
|
|
184
|
+
.rh-top { flex-wrap:wrap; }
|
|
185
|
+
}
|
|
186
|
+
@media (max-width:440px) { .rh-cards { grid-template-columns:1fr; } }
|
|
126
187
|
.rh-field { margin-bottom:16px; max-width:640px; }
|
|
127
188
|
.rh-field label { display:block; font-size:12px; color:var(--muted); margin-bottom:6px; }
|
|
128
189
|
.rh-field input, .rh-field textarea { width:100%; background:var(--panel); border:1px solid var(--line); border-radius:9px; padding:9px 12px; color:var(--text); font:13px var(--mono); }
|
|
@@ -160,6 +221,7 @@
|
|
|
160
221
|
var started = false, lastProcessed = null, lastFailed = null, lastBacklog = null, lastT = null;
|
|
161
222
|
var POLL_MS = <%= (RoundhouseUi.poll_interval.to_f * 1000).round %>;
|
|
162
223
|
var samples = [], buckets = [], bucketStart = null; // samples = current bucket; buckets = finalized per-interval averages
|
|
224
|
+
var lastPts = []; // points last drawn, for the hover tooltip to read
|
|
163
225
|
function setText(id, t) { var el = document.getElementById(id); if (el) el.textContent = t; }
|
|
164
226
|
function humanizeEta(s) {
|
|
165
227
|
if (s < 60) return "~" + Math.round(s) + "s";
|
|
@@ -179,12 +241,28 @@
|
|
|
179
241
|
// finalized buckets + the in-progress bucket as a provisional last point,
|
|
180
242
|
// so the chart shows data immediately instead of waiting a full interval.
|
|
181
243
|
var pts = buckets.slice(-60); if (samples.length) pts = pts.concat([ avg(samples) ]);
|
|
182
|
-
|
|
183
|
-
var
|
|
244
|
+
lastPts = pts;
|
|
245
|
+
var n = pts.length; if (n < 2) { setText("rh-chart-peak", ""); return; }
|
|
246
|
+
var peak = Math.max.apply(null, pts);
|
|
247
|
+
var max = peak * 1.25 || 1;
|
|
184
248
|
var x = function (i) { return i / (n - 1) * w; }, y = function (v) { return h - pad - v / max * (h - pad * 2); };
|
|
185
249
|
var g = ctx.createLinearGradient(0, 0, 0, h); g.addColorStop(0, "rgba(110,139,255,.30)"); g.addColorStop(1, "rgba(110,139,255,0)");
|
|
186
250
|
ctx.beginPath(); ctx.moveTo(0, h); pts.forEach(function (v, i) { ctx.lineTo(x(i), y(v)); }); ctx.lineTo(w, h); ctx.closePath(); ctx.fillStyle = g; ctx.fill();
|
|
187
251
|
ctx.beginPath(); pts.forEach(function (v, i) { i ? ctx.lineTo(x(i), y(v)) : ctx.moveTo(x(i), y(v)); }); ctx.strokeStyle = "#6E8BFF"; ctx.lineWidth = 2; ctx.lineJoin = "round"; ctx.stroke();
|
|
252
|
+
// emphasized endpoint (small marker; canvas is vertically scaled, so a rect reads cleaner than an arc)
|
|
253
|
+
ctx.fillStyle = "#6E8BFF"; ctx.fillRect(x(n - 1) - 3, y(pts[n - 1]) - 3, 6, 6);
|
|
254
|
+
setText("rh-chart-peak", "peak " + Math.round(peak) + "/s");
|
|
255
|
+
}
|
|
256
|
+
// Hover tooltip: map the cursor's fractional x to the nearest drawn point.
|
|
257
|
+
function chartHover(e) {
|
|
258
|
+
var cv = document.getElementById("rh-chart"), tip = document.getElementById("rh-chart-tip");
|
|
259
|
+
if (!cv || !tip || lastPts.length < 2) return;
|
|
260
|
+
var rect = cv.getBoundingClientRect();
|
|
261
|
+
var fx = Math.min(1, Math.max(0, (e.clientX - rect.left) / rect.width));
|
|
262
|
+
var idx = Math.round(fx * (lastPts.length - 1));
|
|
263
|
+
tip.textContent = Math.round(lastPts[idx]) + "/s";
|
|
264
|
+
tip.style.left = (e.clientX - rect.left) + "px";
|
|
265
|
+
tip.hidden = false;
|
|
188
266
|
}
|
|
189
267
|
function apply(d) {
|
|
190
268
|
Object.keys(d).forEach(function (k) {
|
|
@@ -197,6 +275,7 @@
|
|
|
197
275
|
fetch("<%= dashboard_stats_path %>", { headers: { Accept: "application/json" } })
|
|
198
276
|
.then(function (r) { return r.json(); })
|
|
199
277
|
.then(function (d) {
|
|
278
|
+
setConn(true);
|
|
200
279
|
apply(d);
|
|
201
280
|
var now = Date.now();
|
|
202
281
|
var backlog = d.enqueued + d.scheduled + d.retries;
|
|
@@ -228,9 +307,20 @@
|
|
|
228
307
|
}
|
|
229
308
|
lastProcessed = d.processed; lastFailed = d.failed; lastBacklog = backlog; lastT = now;
|
|
230
309
|
})
|
|
231
|
-
.catch(function () {});
|
|
310
|
+
.catch(function () { setConn(false); });
|
|
311
|
+
}
|
|
312
|
+
// Surface a failed poll instead of letting the numbers silently go stale.
|
|
313
|
+
function setConn(ok) {
|
|
314
|
+
var el = document.getElementById("rh-live"), lbl = document.getElementById("rh-live-label");
|
|
315
|
+
if (el) el.classList.toggle("is-stale", !ok);
|
|
316
|
+
if (lbl) lbl.textContent = ok ? "live" : "reconnecting…";
|
|
232
317
|
}
|
|
233
318
|
function startOnce() { if (started) return; started = true; poll(); setInterval(poll, POLL_MS); }
|
|
319
|
+
function wireChart() {
|
|
320
|
+
var cv = document.getElementById("rh-chart"); if (!cv) return;
|
|
321
|
+
cv.onmousemove = chartHover;
|
|
322
|
+
cv.onmouseleave = function () { var t = document.getElementById("rh-chart-tip"); if (t) t.hidden = true; };
|
|
323
|
+
}
|
|
234
324
|
function syncTheme() { var b = document.getElementById("rh-theme"); if (b) b.textContent = document.documentElement.getAttribute("data-theme") === "light" ? "☀" : "☾"; }
|
|
235
325
|
function syncWidth() { var b = document.getElementById("rh-width"); if (b) b.classList.toggle("is-on", document.documentElement.getAttribute("data-width") === "full"); }
|
|
236
326
|
|
|
@@ -342,8 +432,8 @@
|
|
|
342
432
|
else if (e.key === "Enter") { e.preventDefault(); if (palFiltered[palSel]) palRun(palFiltered[palSel]); }
|
|
343
433
|
});
|
|
344
434
|
|
|
345
|
-
document.addEventListener("turbo:load", function () { startOnce(); syncTheme(); syncWidth(); setActiveNav(); restoreChartInterval(); draw(); });
|
|
346
|
-
document.addEventListener("DOMContentLoaded", function () { startOnce(); syncTheme(); syncWidth(); setActiveNav(); restoreChartInterval(); });
|
|
435
|
+
document.addEventListener("turbo:load", function () { startOnce(); syncTheme(); syncWidth(); setActiveNav(); restoreChartInterval(); wireChart(); draw(); });
|
|
436
|
+
document.addEventListener("DOMContentLoaded", function () { startOnce(); syncTheme(); syncWidth(); setActiveNav(); restoreChartInterval(); wireChart(); });
|
|
347
437
|
document.addEventListener("visibilitychange", function () { if (!document.hidden) poll(); });
|
|
348
438
|
})();
|
|
349
439
|
</script>
|
|
@@ -388,7 +478,7 @@
|
|
|
388
478
|
<h1><%= yield :title %></h1>
|
|
389
479
|
<span class="rh-crumb"><%= yield :crumb %></span>
|
|
390
480
|
<span class="rh-spacer"></span>
|
|
391
|
-
<span class="rh-live"><span class="rh-dot"></span> live · <span class="num" data-stat="rate">—</span>/min</span>
|
|
481
|
+
<span class="rh-live" id="rh-live"><span class="rh-dot"></span> <span id="rh-live-label">live</span> · <span class="num" data-stat="rate">—</span>/min</span>
|
|
392
482
|
<% if RoundhouseUi.read_only %><span class="rh-ro">read-only</span><% end %>
|
|
393
483
|
<button class="rh-kbd" id="rh-palette-open" type="button" title="Command palette (⌘K)">⌘K</button>
|
|
394
484
|
<button class="rh-iconbtn" id="rh-width" type="button" title="Toggle full width" aria-label="Toggle full width">⟷</button>
|
|
@@ -1,49 +1,83 @@
|
|
|
1
1
|
<% content_for :title, "Dashboard" %>
|
|
2
2
|
<% content_for :crumb, Rails.env %>
|
|
3
3
|
|
|
4
|
-
<%
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
4
|
+
<% verdict = { ok: "Healthy", warn: "Degraded", crit: "Critical" }[@health.status] %>
|
|
5
|
+
<details class="rh-health rh-health-<%= @health.status %>"<%= " open".html_safe unless @health.healthy? %>>
|
|
6
|
+
<summary>
|
|
7
|
+
<span class="rh-health-dot"></span>
|
|
8
|
+
<span class="rh-health-verdict"><%= verdict %></span>
|
|
9
|
+
<span class="rh-health-reason"><%= @health.reason %></span>
|
|
10
|
+
<span class="rh-health-cta">why ▾</span>
|
|
11
|
+
</summary>
|
|
12
|
+
<div class="rh-health-signals">
|
|
13
|
+
<% @health.signals.each do |s| %>
|
|
14
|
+
<div class="rh-sig">
|
|
15
|
+
<span class="rh-st rh-st-<%= s.status %>"><%= s.status %></span>
|
|
16
|
+
<b><%= s.label %></b>
|
|
17
|
+
<span class="rh-sub"><%= s.detail %></span>
|
|
12
18
|
</div>
|
|
13
19
|
<% end %>
|
|
14
20
|
</div>
|
|
15
|
-
|
|
21
|
+
</details>
|
|
16
22
|
|
|
17
23
|
<div class="rh-cards">
|
|
18
|
-
<div class="rh-card">
|
|
19
|
-
<% if stuck.any? %>
|
|
20
|
-
<span class="pill">⚠ Degraded</span>
|
|
21
|
-
<div class="k"><b style="color:var(--warn)"><%= stuck.first.name %></b> queue over budget</div>
|
|
22
|
-
<% else %>
|
|
23
|
-
<span class="pill ok">✓ Healthy</span>
|
|
24
|
-
<div class="k">all queues within budget</div>
|
|
25
|
-
<% end %>
|
|
26
|
-
</div>
|
|
27
24
|
<div class="rh-card">
|
|
28
25
|
<div class="k">Processed</div>
|
|
29
26
|
<div class="v num" data-stat="processed"><%= number_with_delimiter @stats.processed %></div>
|
|
30
|
-
<div class="d"><span class="num" data-stat="rate">—</span> / min</div>
|
|
27
|
+
<div class="d"><span class="num" data-stat="rate">—</span> / min · <%= ((1 - @metrics.failure_ratio) * 100).round(1) %>% ok</div>
|
|
31
28
|
</div>
|
|
32
29
|
<div class="rh-card">
|
|
33
30
|
<div class="k">Failed · total</div>
|
|
34
31
|
<div class="v num" data-stat="failed"><%= number_with_delimiter @stats.failed %></div>
|
|
35
|
-
<div class="d bad"><span class="num" data-stat="dead"><%= number_with_delimiter @stats.dead_size %></span> dead</div>
|
|
32
|
+
<div class="d bad"><span class="num" data-stat="dead"><%= number_with_delimiter @stats.dead_size %></span> dead · <%= (@metrics.failure_ratio * 100).round(1) %>% rate</div>
|
|
36
33
|
</div>
|
|
37
34
|
<div class="rh-card">
|
|
38
35
|
<div class="k">Busy threads</div>
|
|
39
36
|
<div class="v num" data-stat="busy"><%= @stats.workers_size %></div>
|
|
40
|
-
<div class="d"
|
|
37
|
+
<div class="d"><%= @metrics.utilization ? "#{(@metrics.utilization * 100).round}% of #{@metrics.concurrency} busy" : "no workers reporting" %></div>
|
|
38
|
+
</div>
|
|
39
|
+
<div class="rh-card">
|
|
40
|
+
<div class="k">Backlog</div>
|
|
41
|
+
<div class="v num"><%= number_with_delimiter @metrics.backlog %></div>
|
|
42
|
+
<div class="d">enqueued <span class="num" data-stat="enqueued"><%= number_with_delimiter @stats.enqueued %></span> · scheduled <span class="num" data-stat="scheduled"><%= number_with_delimiter @stats.scheduled_size %></span></div>
|
|
41
43
|
</div>
|
|
42
44
|
</div>
|
|
43
45
|
|
|
44
46
|
<div class="rh-chart-wrap">
|
|
45
|
-
<div class="top"><h3>Throughput</h3><span class="rh-sub">jobs / sec · live</span><span class="now"><span id="rh-chart-now">—</span>/s</span><select id="rh-chart-interval" aria-label="Throughput interval" style="margin-left:12px;background:var(--panel-2);color:var(--muted);border:1px solid var(--line);border-radius:7px;padding:3px 6px;font:12px var(--sans)"><option value="10">per 10s</option><option value="30" selected>per 30s</option><option value="60">per 1m</option><option value="300">per 5m</option></select></div>
|
|
47
|
+
<div class="top"><h3>Throughput</h3><span class="rh-sub">jobs / sec · live</span><span class="rh-sub" id="rh-chart-peak" style="margin-left:10px"></span><span class="now"><span id="rh-chart-now">—</span>/s</span><select id="rh-chart-interval" aria-label="Throughput interval" style="margin-left:12px;background:var(--panel-2);color:var(--muted);border:1px solid var(--line);border-radius:7px;padding:3px 6px;font:12px var(--sans)"><option value="10">per 10s</option><option value="30" selected>per 30s</option><option value="60">per 1m</option><option value="300">per 5m</option></select></div>
|
|
46
48
|
<canvas id="rh-chart" width="1100" height="180"></canvas>
|
|
49
|
+
<div id="rh-chart-tip" hidden></div>
|
|
50
|
+
</div>
|
|
51
|
+
|
|
52
|
+
<div class="rh-insights">
|
|
53
|
+
<div class="rh-panel rh-insight">
|
|
54
|
+
<div class="rh-insight-h">Top failing job classes <%= link_to "all errors →", errors_path, class: "rh-sub" %></div>
|
|
55
|
+
<% if @top_errors.empty? %>
|
|
56
|
+
<div class="rh-empty">No failing jobs 🎉</div>
|
|
57
|
+
<% else %>
|
|
58
|
+
<% @top_errors.each do |g| %>
|
|
59
|
+
<div class="rh-insight-row">
|
|
60
|
+
<div class="rh-insight-main"><%= link_to g[:klass], errors_path(q: g[:klass]), class: "rh-joblink" %> <%= error_trace_link(klass: g[:klass], error: g[:error]) %><br><span class="rh-err"><%= g[:error] %></span></div>
|
|
61
|
+
<span class="rh-pill"><%= number_with_delimiter g[:count] %></span>
|
|
62
|
+
</div>
|
|
63
|
+
<% end %>
|
|
64
|
+
<% end %>
|
|
65
|
+
</div>
|
|
66
|
+
|
|
67
|
+
<div class="rh-panel rh-insight">
|
|
68
|
+
<div class="rh-insight-h">Problem queues <%= link_to "all queues →", queues_path, class: "rh-sub" %></div>
|
|
69
|
+
<% if @problem_queues.empty? %>
|
|
70
|
+
<div class="rh-empty">All queues fresh — nothing backing up.</div>
|
|
71
|
+
<% else %>
|
|
72
|
+
<% @problem_queues.each do |q| %>
|
|
73
|
+
<% label, css = queue_state(q.latency) %>
|
|
74
|
+
<div class="rh-insight-row">
|
|
75
|
+
<div class="rh-insight-main"><%= link_to q.name, queues_path, class: "rh-joblink rh-mono" %><br><span class="rh-sub"><%= number_with_delimiter q.size %> waiting</span></div>
|
|
76
|
+
<span class="rh-st <%= css %>"><%= q.latency < 60 ? "#{q.latency.round(1)}s" : distance_of_time_in_words(0, q.latency) %></span>
|
|
77
|
+
</div>
|
|
78
|
+
<% end %>
|
|
79
|
+
<% end %>
|
|
80
|
+
</div>
|
|
47
81
|
</div>
|
|
48
82
|
|
|
49
83
|
<h2 class="rh-h2">Queues <span class="hint">live · click a queue to manage</span></h2>
|
|
@@ -6,6 +6,14 @@
|
|
|
6
6
|
|
|
7
7
|
<h2 class="rh-h2">Dead set · <%= number_with_delimiter @total %> jobs</h2>
|
|
8
8
|
|
|
9
|
+
<% if @query.present? %>
|
|
10
|
+
<div class="rh-bulkbar">
|
|
11
|
+
<%= button_to "↻ Retry all matching", bulk_all_dead_path, method: :post, params: { op: "retry", q: @query }, class: "rh-btn", form_class: "rh-inline", data: { turbo_confirm: "Retry every dead job matching “#{@query}” (up to #{RoundhouseUi::JobSetBrowsing::BULK_CAP})?" } %>
|
|
12
|
+
<%= button_to "✕ Delete all matching", bulk_all_dead_path, method: :post, params: { op: "delete", q: @query }, class: "rh-btn rh-btn-danger", form_class: "rh-inline", data: { turbo_confirm: "Permanently delete every dead job matching “#{@query}” (up to #{RoundhouseUi::JobSetBrowsing::BULK_CAP})?" } %>
|
|
13
|
+
<span class="rh-sub">acts on every match for “<%= @query %>”, not just this page</span>
|
|
14
|
+
</div>
|
|
15
|
+
<% end %>
|
|
16
|
+
|
|
9
17
|
<%= form_with url: bulk_dead_path, method: :post, data: { turbo: false } do %>
|
|
10
18
|
<div class="rh-bulkbar">
|
|
11
19
|
<button type="submit" name="op" value="retry" class="rh-btn">↻ Retry selected</button>
|
|
@@ -9,7 +9,15 @@
|
|
|
9
9
|
</div>
|
|
10
10
|
|
|
11
11
|
<div class="rh-sec">Arguments<% if RoundhouseUi.redact_args.present? %> <span class="rh-sub">— sensitive keys masked</span><% end %></div>
|
|
12
|
-
|
|
12
|
+
<% args_json = JSON.pretty_generate(RoundhouseUi::Redaction.apply(item["args"] || [])) %>
|
|
13
|
+
<% if args_json.lines.size > 30 %>
|
|
14
|
+
<details class="rh-disclose">
|
|
15
|
+
<summary><%= pluralize(args_json.lines.size, "line") %> — expand arguments</summary>
|
|
16
|
+
<pre class="rh-pre"><%= args_json %></pre>
|
|
17
|
+
</details>
|
|
18
|
+
<% else %>
|
|
19
|
+
<pre class="rh-pre"><%= args_json %></pre>
|
|
20
|
+
<% end %>
|
|
13
21
|
|
|
14
22
|
<% if item["error_class"].present? %>
|
|
15
23
|
<div class="rh-sec">Error</div>
|
|
@@ -17,7 +25,10 @@
|
|
|
17
25
|
<% if item["error_message"].present? %><p class="rh-sub" style="margin:6px 0 0"><%= item["error_message"] %></p><% end %>
|
|
18
26
|
<% bt = item["error_backtrace"] %>
|
|
19
27
|
<% if bt.is_a?(Array) && bt.any? %>
|
|
20
|
-
<
|
|
28
|
+
<details class="rh-disclose"<%= " open".html_safe if bt.size <= 20 %>>
|
|
29
|
+
<summary><%= pluralize(bt.size, "line") %> backtrace</summary>
|
|
30
|
+
<pre class="rh-pre"><%= bt.join("\n") %></pre>
|
|
31
|
+
</details>
|
|
21
32
|
<% end %>
|
|
22
33
|
<% end %>
|
|
23
34
|
|
|
@@ -47,3 +47,24 @@
|
|
|
47
47
|
<div class="d">backlog clears in, at current rate</div>
|
|
48
48
|
</div>
|
|
49
49
|
</div>
|
|
50
|
+
|
|
51
|
+
<% if RoundhouseUi.collect_durations %>
|
|
52
|
+
<h2 class="rh-h2">Slowest job classes <span class="hint">by total time · from RoundhouseUi::DurationCollector</span></h2>
|
|
53
|
+
<% if @durations.blank? %>
|
|
54
|
+
<div class="rh-panel"><div class="rh-empty">No durations recorded yet — jobs need to run with the collector middleware installed.</div></div>
|
|
55
|
+
<% else %>
|
|
56
|
+
<table class="rh-table">
|
|
57
|
+
<thead><tr><th>Job class</th><th class="r">Runs</th><th class="r">Avg</th><th class="r">Total time</th></tr></thead>
|
|
58
|
+
<tbody>
|
|
59
|
+
<% @durations.each do |d| %>
|
|
60
|
+
<tr>
|
|
61
|
+
<td class="rh-mono"><%= d[:klass] %></td>
|
|
62
|
+
<td class="r rh-mono"><%= number_with_delimiter d[:count] %></td>
|
|
63
|
+
<td class="r rh-mono <%= "rh-lat-warn" if d[:avg_ms] >= 1000 %>"><%= d[:avg_ms] >= 1000 ? "#{(d[:avg_ms] / 1000).round(1)}s" : "#{d[:avg_ms].round}ms" %></td>
|
|
64
|
+
<td class="r rh-mono"><%= d[:total_ms] >= 1000 ? "#{(d[:total_ms] / 1000).round(1)}s" : "#{d[:total_ms].round}ms" %></td>
|
|
65
|
+
</tr>
|
|
66
|
+
<% end %>
|
|
67
|
+
</tbody>
|
|
68
|
+
</table>
|
|
69
|
+
<% end %>
|
|
70
|
+
<% end %>
|
|
@@ -5,6 +5,15 @@
|
|
|
5
5
|
</form>
|
|
6
6
|
|
|
7
7
|
<h2 class="rh-h2">Retries · <%= number_with_delimiter @total %> jobs</h2>
|
|
8
|
+
|
|
9
|
+
<% if @query.present? %>
|
|
10
|
+
<div class="rh-bulkbar">
|
|
11
|
+
<%= button_to "↻ Run all matching now", bulk_all_retries_path, method: :post, params: { op: "retry", q: @query }, class: "rh-btn", form_class: "rh-inline", data: { turbo_confirm: "Run every retry matching “#{@query}” now (up to #{RoundhouseUi::JobSetBrowsing::BULK_CAP})?" } %>
|
|
12
|
+
<%= button_to "✕ Delete all matching", bulk_all_retries_path, method: :post, params: { op: "delete", q: @query }, class: "rh-btn rh-btn-danger", form_class: "rh-inline", data: { turbo_confirm: "Permanently delete every retry matching “#{@query}” (up to #{RoundhouseUi::JobSetBrowsing::BULK_CAP})?" } %>
|
|
13
|
+
<span class="rh-sub">acts on every match for “<%= @query %>”, not just this page</span>
|
|
14
|
+
</div>
|
|
15
|
+
<% end %>
|
|
16
|
+
|
|
8
17
|
<table class="rh-table">
|
|
9
18
|
<thead><tr><th>Job</th><th>Last error</th><th class="r">Attempt</th><th class="r">Next try</th><th class="r">Actions</th></tr></thead>
|
|
10
19
|
<tbody>
|
data/config/routes.rb
CHANGED
|
@@ -30,13 +30,15 @@ RoundhouseUi::Engine.routes.draw do
|
|
|
30
30
|
post "scheduled/:jid/delete" => "scheduled#destroy", as: :delete_scheduled
|
|
31
31
|
|
|
32
32
|
get "retries" => "retries#index", as: :retries
|
|
33
|
+
post "retries/bulk_all" => "retries#bulk_all", as: :bulk_all_retries
|
|
33
34
|
post "retries/:jid/run" => "retries#requeue", as: :run_retry
|
|
34
35
|
post "retries/:jid/delete" => "retries#destroy", as: :delete_retry
|
|
35
36
|
|
|
36
37
|
get "dead" => "dead#index", as: :dead_set
|
|
37
|
-
post "dead/bulk" => "dead#bulk",
|
|
38
|
-
post "dead
|
|
39
|
-
post "dead/:jid/
|
|
38
|
+
post "dead/bulk" => "dead#bulk", as: :bulk_dead
|
|
39
|
+
post "dead/bulk_all" => "dead#bulk_all", as: :bulk_all_dead
|
|
40
|
+
post "dead/:jid/retry" => "dead#requeue", as: :retry_dead_job
|
|
41
|
+
post "dead/:jid/delete" => "dead#destroy", as: :delete_dead_job
|
|
40
42
|
|
|
41
43
|
get "errors" => "errors#index", as: :errors
|
|
42
44
|
get "capsules" => "capsules#index", as: :capsules
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
module RoundhouseUi
|
|
2
|
+
# Opt-in server middleware that records per-class execution time, so the UI can
|
|
3
|
+
# answer "which job classes are slow?" — something Sidekiq doesn't track. Two
|
|
4
|
+
# cheap Redis writes per job (a counter + a summed-ms float). Off by default;
|
|
5
|
+
# enable in your Sidekiq server config:
|
|
6
|
+
#
|
|
7
|
+
# Sidekiq.configure_server do |config|
|
|
8
|
+
# config.server_middleware { |chain| chain.add RoundhouseUi::DurationCollector }
|
|
9
|
+
# end
|
|
10
|
+
class DurationCollector
|
|
11
|
+
KEY = "roundhouse:durations".freeze
|
|
12
|
+
|
|
13
|
+
def call(_worker, job, _queue)
|
|
14
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
15
|
+
yield
|
|
16
|
+
ensure
|
|
17
|
+
record(job["class"], (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) * 1000.0)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def record(klass, elapsed_ms)
|
|
23
|
+
return unless klass
|
|
24
|
+
|
|
25
|
+
Sidekiq.redis do |conn|
|
|
26
|
+
conn.call("HINCRBY", KEY, "#{klass}\x00count", 1)
|
|
27
|
+
conn.call("HINCRBYFLOAT", KEY, "#{klass}\x00ms", elapsed_ms)
|
|
28
|
+
end
|
|
29
|
+
rescue => e
|
|
30
|
+
# Metrics collection must never break a job.
|
|
31
|
+
Sidekiq.logger&.warn("[roundhouse] duration collect failed: #{e.message}")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# [{ klass:, count:, total_ms:, avg_ms: }], slowest (by total time) first.
|
|
35
|
+
def self.summary(limit: 20)
|
|
36
|
+
raw = Sidekiq.redis { |conn| conn.call("HGETALL", KEY) }
|
|
37
|
+
pairs = raw.is_a?(Hash) ? raw : raw.each_slice(2).to_a # redis-client: flat array; redis-rb: hash
|
|
38
|
+
|
|
39
|
+
by_class = Hash.new { |h, k| h[k] = { klass: k, count: 0, total_ms: 0.0 } }
|
|
40
|
+
pairs.each do |field, value|
|
|
41
|
+
klass, kind = field.split("\x00", 2)
|
|
42
|
+
next unless kind
|
|
43
|
+
|
|
44
|
+
kind == "count" ? by_class[klass][:count] = value.to_i : by_class[klass][:total_ms] = value.to_f
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
by_class.values
|
|
48
|
+
.map { |r| r.merge(avg_ms: r[:count].positive? ? r[:total_ms] / r[:count] : 0.0) }
|
|
49
|
+
.sort_by { |r| -r[:total_ms] }
|
|
50
|
+
.first(limit)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
require "sidekiq/api"
|
|
2
|
+
|
|
3
|
+
module RoundhouseUi
|
|
4
|
+
# Groups failing jobs across the retry + dead sets (and the sidekiq-failures
|
|
5
|
+
# `failed` set, when opted in) by a fingerprint of job class + error class —
|
|
6
|
+
# so one bad deploy reads as a single issue with a count, not thousands of
|
|
7
|
+
# identical rows. Used by the Errors page and the dashboard's "top failing"
|
|
8
|
+
# panel, so the aggregation lives here rather than in a controller.
|
|
9
|
+
class ErrorGroups
|
|
10
|
+
DEFAULT_SCAN_LIMIT = 1_000 # cap entries scanned per pass; surfaced honestly
|
|
11
|
+
|
|
12
|
+
Result = Struct.new(:groups, :scanned, :truncated, keyword_init: true)
|
|
13
|
+
|
|
14
|
+
def initialize(query: nil, limit: DEFAULT_SCAN_LIMIT)
|
|
15
|
+
@query = query.to_s.strip
|
|
16
|
+
@limit = limit
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def call
|
|
20
|
+
groups = {}
|
|
21
|
+
scanned = 0
|
|
22
|
+
truncated = false
|
|
23
|
+
|
|
24
|
+
sources.each do |source, set|
|
|
25
|
+
set.each do |entry|
|
|
26
|
+
scanned += 1
|
|
27
|
+
if scanned > @limit
|
|
28
|
+
truncated = true
|
|
29
|
+
break
|
|
30
|
+
end
|
|
31
|
+
record(groups, source, entry)
|
|
32
|
+
end
|
|
33
|
+
break if truncated
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
list = groups.values.sort_by { |g| -g[:count] }
|
|
37
|
+
list = list.select { |g| "#{g[:klass]} #{g[:error]}".downcase.include?(@query.downcase) } if @query.present?
|
|
38
|
+
Result.new(groups: list, scanned: scanned, truncated: truncated)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
# Sidekiq's native sets, plus the sidekiq-failures `failed` set when opted in
|
|
44
|
+
# and loaded. Its FailureSet is a Sidekiq::JobSet, so it iterates like the rest.
|
|
45
|
+
def sources
|
|
46
|
+
sets = { "retry" => Sidekiq::RetrySet.new, "dead" => Sidekiq::DeadSet.new }
|
|
47
|
+
if RoundhouseUi.show_sidekiq_failures && defined?(Sidekiq::Failures::FailureSet)
|
|
48
|
+
sets["failed"] = Sidekiq::Failures::FailureSet.new
|
|
49
|
+
end
|
|
50
|
+
sets
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def record(groups, source, entry)
|
|
54
|
+
error = entry.item["error_class"] || "UnknownError"
|
|
55
|
+
group = (groups["#{entry.klass}|#{error}"] ||= {
|
|
56
|
+
klass: entry.klass, error: error, count: 0, last_at: nil, queues: [], sources: []
|
|
57
|
+
})
|
|
58
|
+
group[:count] += 1
|
|
59
|
+
group[:queues] |= [ entry.queue ]
|
|
60
|
+
group[:sources] |= [ source ]
|
|
61
|
+
at = entry.at
|
|
62
|
+
group[:last_at] = at if at && (group[:last_at].nil? || at > group[:last_at])
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
module RoundhouseUi
|
|
2
|
+
# A composite health verdict for the dashboard. Instead of a static green dot,
|
|
3
|
+
# it rolls up the signals an on-call engineer actually checks — error rate,
|
|
4
|
+
# queue latency, worker utilization — into one status + a human reason, and
|
|
5
|
+
# exposes the sub-signals so the banner can explain *why*.
|
|
6
|
+
class Health
|
|
7
|
+
Signal = Struct.new(:key, :label, :status, :detail, keyword_init: true)
|
|
8
|
+
|
|
9
|
+
RANK = { ok: 0, warn: 1, crit: 2 }.freeze
|
|
10
|
+
|
|
11
|
+
def initialize(stats:, queues:, metrics:)
|
|
12
|
+
@stats = stats
|
|
13
|
+
@queues = queues
|
|
14
|
+
@metrics = metrics
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def signals
|
|
18
|
+
@signals ||= [ error_rate_signal, latency_signal, utilization_signal ].compact
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Worst sub-signal wins.
|
|
22
|
+
def status
|
|
23
|
+
signals.map(&:status).max_by { |s| RANK[s] } || :ok
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def reason
|
|
27
|
+
worst = signals.max_by { |s| RANK[s.status] }
|
|
28
|
+
return "all signals nominal" if worst.nil? || worst.status == :ok
|
|
29
|
+
|
|
30
|
+
worst.detail
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def healthy?
|
|
34
|
+
status == :ok
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
def error_rate_signal
|
|
40
|
+
ratio = @metrics.failure_ratio
|
|
41
|
+
status = if ratio >= 0.10 then :crit elsif ratio >= 0.02 then :warn else :ok end
|
|
42
|
+
Signal.new(key: "error_rate", label: "Error rate (lifetime)", status: status,
|
|
43
|
+
detail: "#{(ratio * 100).round(1)}% of processed jobs have failed")
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def latency_signal
|
|
47
|
+
worst = @queues.max_by(&:latency)
|
|
48
|
+
return Signal.new(key: "latency", label: "Queue latency", status: :ok, detail: "no active queues") if worst.nil?
|
|
49
|
+
|
|
50
|
+
lat = worst.latency
|
|
51
|
+
status = if lat > 600 then :crit elsif lat > 60 then :warn else :ok end
|
|
52
|
+
detail = status == :ok ? "all queues fresh (< 60s)" : "#{worst.name}: oldest job #{lat.round}s"
|
|
53
|
+
Signal.new(key: "latency", label: "Queue latency", status: status, detail: detail)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def utilization_signal
|
|
57
|
+
util = @metrics.utilization
|
|
58
|
+
return nil if util.nil? # no processes reporting in — can't judge
|
|
59
|
+
|
|
60
|
+
status = if util >= 1.0 then :crit elsif util >= 0.85 then :warn else :ok end
|
|
61
|
+
Signal.new(key: "utilization", label: "Worker utilization", status: status,
|
|
62
|
+
detail: "#{(util * 100).round}% of worker threads busy")
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
data/lib/roundhouse_ui.rb
CHANGED
|
@@ -10,6 +10,9 @@ require "roundhouse_ui/redaction"
|
|
|
10
10
|
require "roundhouse_ui/cancellation"
|
|
11
11
|
require "roundhouse_ui/cancel_middleware"
|
|
12
12
|
require "roundhouse_ui/metrics"
|
|
13
|
+
require "roundhouse_ui/error_groups"
|
|
14
|
+
require "roundhouse_ui/health"
|
|
15
|
+
require "roundhouse_ui/duration_collector"
|
|
13
16
|
|
|
14
17
|
# Brand name is "Roundhouse"; the gem and Ruby namespace are RoundhouseUi
|
|
15
18
|
# (matching the published gem name `roundhouse_ui`).
|
|
@@ -57,6 +60,12 @@ module RoundhouseUi
|
|
|
57
60
|
# Sidekiq's retry/dead sets, so this is the only way to surface them here.
|
|
58
61
|
attr_accessor :show_sidekiq_failures
|
|
59
62
|
|
|
63
|
+
# Opt-in: record per-class job durations (via RoundhouseUi::DurationCollector
|
|
64
|
+
# server middleware) so the Metrics page can show the slowest job classes.
|
|
65
|
+
# Default false; reads/writes a single Redis hash. The flag gates the UI; the
|
|
66
|
+
# collection itself is enabled by installing the middleware.
|
|
67
|
+
attr_accessor :collect_durations
|
|
68
|
+
|
|
60
69
|
# Seconds between dashboard stat polls. Lower = livelier, but each poll also
|
|
61
70
|
# re-runs the host's auth/routing on the mount, so a busy console can add DB
|
|
62
71
|
# load. Default 5s; raise it if polling shows up in your traces.
|
|
@@ -91,4 +100,5 @@ module RoundhouseUi
|
|
|
91
100
|
self.show_sidekiq_failures = false
|
|
92
101
|
self.pause_enabled = true
|
|
93
102
|
self.poll_interval = 5
|
|
103
|
+
self.collect_durations = false
|
|
94
104
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: roundhouse_ui
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.8.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- R.J. Robinson
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-
|
|
11
|
+
date: 2026-07-01 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rails
|
|
@@ -96,8 +96,11 @@ files:
|
|
|
96
96
|
- lib/roundhouse_ui/audit.rb
|
|
97
97
|
- lib/roundhouse_ui/cancel_middleware.rb
|
|
98
98
|
- lib/roundhouse_ui/cancellation.rb
|
|
99
|
+
- lib/roundhouse_ui/duration_collector.rb
|
|
99
100
|
- lib/roundhouse_ui/engine.rb
|
|
101
|
+
- lib/roundhouse_ui/error_groups.rb
|
|
100
102
|
- lib/roundhouse_ui/fetch.rb
|
|
103
|
+
- lib/roundhouse_ui/health.rb
|
|
101
104
|
- lib/roundhouse_ui/metrics.rb
|
|
102
105
|
- lib/roundhouse_ui/observability.rb
|
|
103
106
|
- lib/roundhouse_ui/pause.rb
|