roundhouse_ui 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/MIT-LICENSE +20 -0
- data/README.md +166 -0
- data/Rakefile +3 -0
- data/app/assets/javascripts/roundhouse_ui/turbo.min.js +35 -0
- data/app/assets/stylesheets/roundhouse_ui/application.css +15 -0
- data/app/controllers/concerns/roundhouse_ui/job_set_browsing.rb +41 -0
- data/app/controllers/roundhouse_ui/application_controller.rb +75 -0
- data/app/controllers/roundhouse_ui/assets_controller.rb +16 -0
- data/app/controllers/roundhouse_ui/audit_controller.rb +7 -0
- data/app/controllers/roundhouse_ui/busy_controller.rb +29 -0
- data/app/controllers/roundhouse_ui/capsules_controller.rb +27 -0
- data/app/controllers/roundhouse_ui/dashboard_controller.rb +26 -0
- data/app/controllers/roundhouse_ui/dead_controller.rb +46 -0
- data/app/controllers/roundhouse_ui/errors_controller.rb +50 -0
- data/app/controllers/roundhouse_ui/jobs_controller.rb +94 -0
- data/app/controllers/roundhouse_ui/metrics_controller.rb +8 -0
- data/app/controllers/roundhouse_ui/queues_controller.rb +40 -0
- data/app/controllers/roundhouse_ui/redis_controller.rb +21 -0
- data/app/controllers/roundhouse_ui/retries_controller.rb +34 -0
- data/app/controllers/roundhouse_ui/scheduled_controller.rb +34 -0
- data/app/controllers/roundhouse_ui/snapshots_controller.rb +26 -0
- data/app/controllers/roundhouse_ui/workers_controller.rb +33 -0
- data/app/helpers/roundhouse_ui/application_helper.rb +4 -0
- data/app/helpers/roundhouse_ui/nav_helper.rb +24 -0
- data/app/helpers/roundhouse_ui/observability_helper.rb +13 -0
- data/app/views/layouts/roundhouse_ui/application.html.erb +365 -0
- data/app/views/roundhouse_ui/audit/index.html.erb +21 -0
- data/app/views/roundhouse_ui/busy/index.html.erb +23 -0
- data/app/views/roundhouse_ui/capsules/index.html.erb +22 -0
- data/app/views/roundhouse_ui/dashboard/show.html.erb +68 -0
- data/app/views/roundhouse_ui/dead/index.html.erb +46 -0
- data/app/views/roundhouse_ui/errors/index.html.erb +28 -0
- data/app/views/roundhouse_ui/jobs/_form.html.erb +24 -0
- data/app/views/roundhouse_ui/jobs/edit.html.erb +2 -0
- data/app/views/roundhouse_ui/jobs/new.html.erb +2 -0
- data/app/views/roundhouse_ui/jobs/show.html.erb +33 -0
- data/app/views/roundhouse_ui/metrics/show.html.erb +49 -0
- data/app/views/roundhouse_ui/queues/index.html.erb +45 -0
- data/app/views/roundhouse_ui/redis/show.html.erb +59 -0
- data/app/views/roundhouse_ui/retries/index.html.erb +33 -0
- data/app/views/roundhouse_ui/scheduled/index.html.erb +29 -0
- data/app/views/roundhouse_ui/shared/_pager.html.erb +15 -0
- data/app/views/roundhouse_ui/snapshots/index.html.erb +25 -0
- data/app/views/roundhouse_ui/workers/index.html.erb +39 -0
- data/config/routes.rb +54 -0
- data/lib/roundhouse_ui/audit.rb +25 -0
- data/lib/roundhouse_ui/cancel_middleware.rb +19 -0
- data/lib/roundhouse_ui/cancellation.rb +37 -0
- data/lib/roundhouse_ui/engine.rb +5 -0
- data/lib/roundhouse_ui/fetch.rb +36 -0
- data/lib/roundhouse_ui/metrics.rb +51 -0
- data/lib/roundhouse_ui/observability.rb +46 -0
- data/lib/roundhouse_ui/pause.rb +59 -0
- data/lib/roundhouse_ui/redaction.rb +33 -0
- data/lib/roundhouse_ui/snapshots.rb +90 -0
- data/lib/roundhouse_ui/version.rb +3 -0
- data/lib/roundhouse_ui.rb +73 -0
- data/lib/tasks/roundhouse_ui_tasks.rake +4 -0
- metadata +131 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
<% content_for :title, @entry.item["class"] %>
|
|
2
|
+
<% item = @entry.item %>
|
|
3
|
+
|
|
4
|
+
<div class="rh-tags">
|
|
5
|
+
<span class="rh-tag"><%= @set %></span>
|
|
6
|
+
<span class="rh-tag">queue <%= @entry.queue %></span>
|
|
7
|
+
<% if item["retry_count"] %><span class="rh-tag">retry <%= item["retry_count"] %></span><% end %>
|
|
8
|
+
<span class="rh-tag rh-mono"><%= @entry.jid %></span>
|
|
9
|
+
</div>
|
|
10
|
+
|
|
11
|
+
<div class="rh-sec">Arguments<% if RoundhouseUi.redact_args.present? %> <span class="rh-sub">— sensitive keys masked</span><% end %></div>
|
|
12
|
+
<pre class="rh-pre"><%= JSON.pretty_generate(RoundhouseUi::Redaction.apply(item["args"] || [])) %></pre>
|
|
13
|
+
|
|
14
|
+
<% if item["error_class"].present? %>
|
|
15
|
+
<div class="rh-sec">Error</div>
|
|
16
|
+
<div class="rh-err"><%= item["error_class"] %></div>
|
|
17
|
+
<% if item["error_message"].present? %><p class="rh-sub" style="margin:6px 0 0"><%= item["error_message"] %></p><% end %>
|
|
18
|
+
<% bt = item["error_backtrace"] %>
|
|
19
|
+
<% if bt.is_a?(Array) && bt.any? %>
|
|
20
|
+
<pre class="rh-pre"><%= bt.first(20).join("\n") %></pre>
|
|
21
|
+
<% end %>
|
|
22
|
+
<% end %>
|
|
23
|
+
|
|
24
|
+
<% trace = trace_link(klass: item["class"], jid: @entry.jid, queue: @entry.queue) %>
|
|
25
|
+
<% if trace.present? %>
|
|
26
|
+
<div class="rh-sec">Observability</div>
|
|
27
|
+
<%= trace %>
|
|
28
|
+
<% end %>
|
|
29
|
+
|
|
30
|
+
<div class="rh-pager">
|
|
31
|
+
<% if RoundhouseUi.allow_job_editing %><%= link_to "Edit", edit_job_path(set: @set, jid: @entry.jid), class: "rh-btn rh-btn-primary" %><% end %>
|
|
32
|
+
<%= link_to "← Back", { "dead" => dead_set_path, "retry" => retries_path, "scheduled" => scheduled_path }[@set] || root_path, class: "rh-btn" %>
|
|
33
|
+
</div>
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
<% content_for :title, "Metrics" %>
|
|
2
|
+
|
|
3
|
+
<h2 class="rh-h2">Capacity & reliability <span class="hint">live snapshot</span></h2>
|
|
4
|
+
<div class="rh-cards">
|
|
5
|
+
<div class="rh-card">
|
|
6
|
+
<div class="k">Utilization</div>
|
|
7
|
+
<div class="v num" style="<%= "color:var(--warn)" if @metrics.utilization && @metrics.utilization >= 0.9 %>"><%= @metrics.utilization ? "#{(@metrics.utilization * 100).round}%" : "—" %></div>
|
|
8
|
+
<div class="d"><span class="num"><%= @metrics.busy %></span> / <%= @metrics.concurrency %> threads</div>
|
|
9
|
+
</div>
|
|
10
|
+
<div class="rh-card">
|
|
11
|
+
<div class="k">Idle headroom</div>
|
|
12
|
+
<div class="v num"><%= @metrics.headroom %></div>
|
|
13
|
+
<div class="d">worker threads free</div>
|
|
14
|
+
</div>
|
|
15
|
+
<div class="rh-card">
|
|
16
|
+
<div class="k">Backlog</div>
|
|
17
|
+
<div class="v num"><%= number_with_delimiter @metrics.backlog %></div>
|
|
18
|
+
<div class="d">enqueued + scheduled + retry</div>
|
|
19
|
+
</div>
|
|
20
|
+
<div class="rh-card">
|
|
21
|
+
<div class="k">Failure ratio</div>
|
|
22
|
+
<div class="v num"><%= (@metrics.failure_ratio * 100).round(2) %>%</div>
|
|
23
|
+
<div class="d">failed / processed · lifetime</div>
|
|
24
|
+
</div>
|
|
25
|
+
</div>
|
|
26
|
+
|
|
27
|
+
<h2 class="rh-h2">Live rates <span class="hint">computed client-side · last few seconds</span></h2>
|
|
28
|
+
<div class="rh-cards">
|
|
29
|
+
<div class="rh-card">
|
|
30
|
+
<div class="k">Throughput</div>
|
|
31
|
+
<div class="v num" id="rh-m-throughput">—</div>
|
|
32
|
+
<div class="d">jobs / sec · completed</div>
|
|
33
|
+
</div>
|
|
34
|
+
<div class="rh-card">
|
|
35
|
+
<div class="k">Failure rate</div>
|
|
36
|
+
<div class="v num" id="rh-m-failrate">—</div>
|
|
37
|
+
<div class="d">share of recent jobs</div>
|
|
38
|
+
</div>
|
|
39
|
+
<div class="rh-card">
|
|
40
|
+
<div class="k">Backlog velocity</div>
|
|
41
|
+
<div class="v num" id="rh-m-velocity">—</div>
|
|
42
|
+
<div class="d">+ growing · − draining</div>
|
|
43
|
+
</div>
|
|
44
|
+
<div class="rh-card">
|
|
45
|
+
<div class="k">Burn-down ETA</div>
|
|
46
|
+
<div class="v num" id="rh-m-eta">—</div>
|
|
47
|
+
<div class="d">backlog clears in, at current rate</div>
|
|
48
|
+
</div>
|
|
49
|
+
</div>
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
<% content_for :title, "Queues" %>
|
|
2
|
+
|
|
3
|
+
<% if RoundhouseUi.allow_job_editing %>
|
|
4
|
+
<div class="rh-toolbar"><%= link_to "+ Enqueue job", new_job_path, class: "rh-btn rh-btn-primary" %></div>
|
|
5
|
+
<% end %>
|
|
6
|
+
|
|
7
|
+
<% unless @fetch_installed %>
|
|
8
|
+
<div class="rh-warn">
|
|
9
|
+
Pausing is <strong>recorded but not enforced</strong> — no Roundhouse fetcher has reported in.
|
|
10
|
+
Add it to your Sidekiq config: <code>config[:fetch_class] = RoundhouseUi::Fetch</code>
|
|
11
|
+
</div>
|
|
12
|
+
<% end %>
|
|
13
|
+
|
|
14
|
+
<h2 class="rh-h2">Queues</h2>
|
|
15
|
+
<table class="rh-table">
|
|
16
|
+
<thead><tr><th>Queue</th><th>State</th><th class="r">Size</th><th class="r">Latency</th><th class="r">Actions</th></tr></thead>
|
|
17
|
+
<tbody>
|
|
18
|
+
<% if @queues.empty? %>
|
|
19
|
+
<tr><td colspan="5" class="rh-empty">No active queues</td></tr>
|
|
20
|
+
<% else %>
|
|
21
|
+
<% @queues.each do |q| %>
|
|
22
|
+
<% paused = @paused.include?(q.name) %>
|
|
23
|
+
<tr>
|
|
24
|
+
<td><%= q.name %></td>
|
|
25
|
+
<td><% if paused %><span class="rh-pill rh-pill-warn">paused</span><% else %><span class="rh-sub">active</span><% end %></td>
|
|
26
|
+
<td class="r"><%= number_with_delimiter q.size %></td>
|
|
27
|
+
<td class="r <%= "rh-lat-warn" if q.latency > 60 %>"><%= q.latency.round(2) %>s</td>
|
|
28
|
+
<td class="r">
|
|
29
|
+
<% if paused %>
|
|
30
|
+
<%= button_to "Resume", resume_queue_path(q.name), method: :post, class: "rh-btn", form_class: "rh-inline" %>
|
|
31
|
+
<% else %>
|
|
32
|
+
<%= button_to "Pause", pause_queue_path(q.name), method: :post, class: "rh-btn", form_class: "rh-inline" %>
|
|
33
|
+
<% end %>
|
|
34
|
+
<%= button_to "Snapshot", snapshot_queue_path(q.name), method: :post, class: "rh-btn", form_class: "rh-inline" %>
|
|
35
|
+
<%= button_to "Purge", purge_queue_path(q.name),
|
|
36
|
+
method: :post,
|
|
37
|
+
class: "rh-btn rh-btn-danger",
|
|
38
|
+
form_class: "rh-inline",
|
|
39
|
+
data: { turbo_confirm: "Purge “#{q.name}”? This permanently deletes #{number_with_delimiter q.size} jobs. Snapshot first if you might need them back." } %>
|
|
40
|
+
</td>
|
|
41
|
+
</tr>
|
|
42
|
+
<% end %>
|
|
43
|
+
<% end %>
|
|
44
|
+
</tbody>
|
|
45
|
+
</table>
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
<% content_for :title, "Redis" %>
|
|
2
|
+
|
|
3
|
+
<% policy = @info["maxmemory_policy"] %>
|
|
4
|
+
<% maxmem = @info["maxmemory"].to_i %>
|
|
5
|
+
<% used = @info["used_memory"].to_i %>
|
|
6
|
+
<% evicted = @info["evicted_keys"].to_i %>
|
|
7
|
+
<% risk = maxmem.positive? && policy.present? && policy != "noeviction" %>
|
|
8
|
+
<% hits = @info["keyspace_hits"].to_f %>
|
|
9
|
+
<% misses = @info["keyspace_misses"].to_f %>
|
|
10
|
+
<% hit_rate = (hits + misses).positive? ? (hits / (hits + misses) * 100).round(1) : 100.0 %>
|
|
11
|
+
<% frag = @info["mem_fragmentation_ratio"].to_f %>
|
|
12
|
+
<% uptime = @info["uptime_in_seconds"].to_i %>
|
|
13
|
+
<% keys = @info.to_a.find { |k, _| k.start_with?("db") }&.last.to_s[/keys=(\d+)/, 1] %>
|
|
14
|
+
|
|
15
|
+
<% if risk %>
|
|
16
|
+
<div class="rh-alert">
|
|
17
|
+
<span class="msg">Eviction policy is <b><%= policy %></b> with a memory cap — Sidekiq jobs can be <strong>evicted and silently lost</strong>. Use <b>noeviction</b> on the Redis that backs Sidekiq.</span>
|
|
18
|
+
</div>
|
|
19
|
+
<% elsif evicted.positive? %>
|
|
20
|
+
<div class="rh-alert warn"><span class="msg"><b><%= number_with_delimiter evicted %></b> keys have been evicted on this instance.</span></div>
|
|
21
|
+
<% end %>
|
|
22
|
+
|
|
23
|
+
<div class="rh-cards">
|
|
24
|
+
<div class="rh-card">
|
|
25
|
+
<div class="k">Memory used</div>
|
|
26
|
+
<div class="v num"><%= @info["used_memory_human"] || number_to_human_size(used) %></div>
|
|
27
|
+
<div class="d"><%= maxmem.positive? ? "#{(used.to_f / maxmem * 100).round}% of #{number_to_human_size(maxmem)}" : "no cap set" %></div>
|
|
28
|
+
</div>
|
|
29
|
+
<div class="rh-card">
|
|
30
|
+
<div class="k">Ops / sec</div>
|
|
31
|
+
<div class="v num"><%= number_with_delimiter @info["instantaneous_ops_per_sec"] %></div>
|
|
32
|
+
<div class="d"><%= number_with_delimiter @info["total_commands_processed"] %> total</div>
|
|
33
|
+
</div>
|
|
34
|
+
<div class="rh-card">
|
|
35
|
+
<div class="k">Clients</div>
|
|
36
|
+
<div class="v num"><%= @info["connected_clients"] %></div>
|
|
37
|
+
<div class="d"><%= @info["blocked_clients"] %> blocked</div>
|
|
38
|
+
</div>
|
|
39
|
+
<div class="rh-card">
|
|
40
|
+
<div class="k">Eviction policy</div>
|
|
41
|
+
<div class="v" style="font-size:18px; color:<%= risk ? "var(--crit)" : "var(--good)" %>"><%= policy || "—" %></div>
|
|
42
|
+
<div class="d"><%= evicted.positive? ? "#{number_with_delimiter evicted} evicted" : "0 evicted" %></div>
|
|
43
|
+
</div>
|
|
44
|
+
</div>
|
|
45
|
+
|
|
46
|
+
<h2 class="rh-h2">Signals <span class="hint">from <span class="rh-mono">INFO</span></span></h2>
|
|
47
|
+
<table class="rh-table">
|
|
48
|
+
<tbody>
|
|
49
|
+
<tr><td>Fragmentation ratio</td><td class="r rh-mono <%= "rh-lat-warn" if frag > 1.5 %>"><%= frag.round(2) %>×</td></tr>
|
|
50
|
+
<tr><td>Evicted keys</td><td class="r rh-mono" style="<%= "color:var(--crit)" if evicted.positive? %>"><%= number_with_delimiter evicted %></td></tr>
|
|
51
|
+
<tr><td>Expired keys</td><td class="r rh-mono"><%= number_with_delimiter @info["expired_keys"] %></td></tr>
|
|
52
|
+
<tr><td>Keyspace hit rate</td><td class="r rh-mono"><%= hit_rate %>%</td></tr>
|
|
53
|
+
<tr><td>Blocked clients</td><td class="r rh-mono"><%= @info["blocked_clients"] %></td></tr>
|
|
54
|
+
<tr><td>Rejected connections</td><td class="r rh-mono" style="<%= "color:var(--crit)" if @info["rejected_connections"].to_i.positive? %>"><%= @info["rejected_connections"] %></td></tr>
|
|
55
|
+
<tr><td>Keys (db0)</td><td class="r rh-mono"><%= keys ? number_with_delimiter(keys) : "—" %></td></tr>
|
|
56
|
+
<tr><td>Uptime</td><td class="r rh-mono"><%= uptime >= 86_400 ? "#{uptime / 86_400}d" : "#{uptime / 3_600}h" %></td></tr>
|
|
57
|
+
<tr><td>Redis</td><td class="r rh-mono">v<%= @info["redis_version"] %> · <%= @info["role"] %></td></tr>
|
|
58
|
+
</tbody>
|
|
59
|
+
</table>
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
<% content_for :title, "Retries" %>
|
|
2
|
+
|
|
3
|
+
<form method="get" action="<%= retries_path %>" class="rh-search">
|
|
4
|
+
<input type="search" name="q" value="<%= @query %>" placeholder="search class, jid, error, or arg value…">
|
|
5
|
+
</form>
|
|
6
|
+
|
|
7
|
+
<h2 class="rh-h2">Retries · <%= number_with_delimiter @total %> jobs</h2>
|
|
8
|
+
<table class="rh-table">
|
|
9
|
+
<thead><tr><th>Job</th><th>Last error</th><th class="r">Attempt</th><th class="r">Next try</th><th class="r">Actions</th></tr></thead>
|
|
10
|
+
<tbody>
|
|
11
|
+
<% if @jobs.empty? %>
|
|
12
|
+
<tr><td colspan="5" class="rh-empty"><%= @query.present? ? "No retries match “#{@query}”." : "Nothing waiting to retry 🎉" %></td></tr>
|
|
13
|
+
<% else %>
|
|
14
|
+
<% @jobs.each do |job| %>
|
|
15
|
+
<tr>
|
|
16
|
+
<td><%= link_to job.klass, job_path(set: "retry", jid: job.jid), class: "rh-joblink" %><br><span class="rh-sub"><%= job.jid %></span> <%= trace_link(klass: job.klass, jid: job.jid, queue: job.queue) %></td>
|
|
17
|
+
<td>
|
|
18
|
+
<span class="rh-err"><%= job.item["error_class"] %></span>
|
|
19
|
+
<% if job.item["error_message"].present? %><br><span class="rh-sub"><%= truncate(job.item["error_message"], length: 90) %></span><% end %>
|
|
20
|
+
</td>
|
|
21
|
+
<td class="r">#<%= job.item["retry_count"].to_i + 1 %></td>
|
|
22
|
+
<td class="r"><span class="rh-sub"><%= job.at && job.at > Time.now ? "in #{distance_of_time_in_words(Time.now, job.at)}" : "now" %></span></td>
|
|
23
|
+
<td class="r">
|
|
24
|
+
<% if RoundhouseUi.allow_job_editing %><%= link_to "Edit", edit_job_path(set: "retry", jid: job.jid), class: "rh-btn" %><% end %>
|
|
25
|
+
<%= button_to "Run now", run_retry_path(job.jid), method: :post, class: "rh-btn", form_class: "rh-inline" %>
|
|
26
|
+
<%= button_to "Delete", delete_retry_path(job.jid), method: :post, class: "rh-btn rh-btn-danger", form_class: "rh-inline", data: { turbo_confirm: "Delete #{job.jid}?" } %>
|
|
27
|
+
</td>
|
|
28
|
+
</tr>
|
|
29
|
+
<% end %>
|
|
30
|
+
<% end %>
|
|
31
|
+
</tbody>
|
|
32
|
+
</table>
|
|
33
|
+
<%= render "roundhouse_ui/shared/pager" %>
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
<% content_for :title, "Scheduled" %>
|
|
2
|
+
|
|
3
|
+
<form method="get" action="<%= scheduled_path %>" class="rh-search">
|
|
4
|
+
<input type="search" name="q" value="<%= @query %>" placeholder="search class, jid, or arg value…">
|
|
5
|
+
</form>
|
|
6
|
+
|
|
7
|
+
<h2 class="rh-h2">Scheduled · <%= number_with_delimiter @total %> jobs</h2>
|
|
8
|
+
<table class="rh-table">
|
|
9
|
+
<thead><tr><th>Job</th><th>Queue</th><th class="r">Runs</th><th class="r">Actions</th></tr></thead>
|
|
10
|
+
<tbody>
|
|
11
|
+
<% if @jobs.empty? %>
|
|
12
|
+
<tr><td colspan="4" class="rh-empty"><%= @query.present? ? "No scheduled jobs match “#{@query}”." : "Nothing scheduled" %></td></tr>
|
|
13
|
+
<% else %>
|
|
14
|
+
<% @jobs.each do |job| %>
|
|
15
|
+
<tr>
|
|
16
|
+
<td><%= link_to job.klass, job_path(set: "scheduled", jid: job.jid), class: "rh-joblink" %><br><span class="rh-sub"><%= job.jid %></span></td>
|
|
17
|
+
<td><span class="rh-sub"><%= job.queue %></span></td>
|
|
18
|
+
<td class="r"><span class="rh-sub"><%= job.at && job.at > Time.now ? "in #{distance_of_time_in_words(Time.now, job.at)}" : "now (overdue)" %></span></td>
|
|
19
|
+
<td class="r">
|
|
20
|
+
<% if RoundhouseUi.allow_job_editing %><%= link_to "Edit", edit_job_path(set: "scheduled", jid: job.jid), class: "rh-btn" %><% end %>
|
|
21
|
+
<%= button_to "Enqueue now", enqueue_scheduled_path(job.jid), method: :post, class: "rh-btn", form_class: "rh-inline" %>
|
|
22
|
+
<%= button_to "Delete", delete_scheduled_path(job.jid), method: :post, class: "rh-btn rh-btn-danger", form_class: "rh-inline", data: { turbo_confirm: "Delete #{job.jid}?" } %>
|
|
23
|
+
</td>
|
|
24
|
+
</tr>
|
|
25
|
+
<% end %>
|
|
26
|
+
<% end %>
|
|
27
|
+
</tbody>
|
|
28
|
+
</table>
|
|
29
|
+
<%= render "roundhouse_ui/shared/pager" %>
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
<% if @page > 1 || @has_next %>
|
|
2
|
+
<div class="rh-pager">
|
|
3
|
+
<% if @page > 1 %>
|
|
4
|
+
<%= link_to "← Prev", url_for(page: @page - 1, q: @query.presence), class: "rh-btn" %>
|
|
5
|
+
<% else %>
|
|
6
|
+
<span class="rh-btn" style="opacity:.4; pointer-events:none">← Prev</span>
|
|
7
|
+
<% end %>
|
|
8
|
+
<span class="rh-sub">Page <%= @page %></span>
|
|
9
|
+
<% if @has_next %>
|
|
10
|
+
<%= link_to "Next →", url_for(page: @page + 1, q: @query.presence), class: "rh-btn" %>
|
|
11
|
+
<% else %>
|
|
12
|
+
<span class="rh-btn" style="opacity:.4; pointer-events:none">Next →</span>
|
|
13
|
+
<% end %>
|
|
14
|
+
</div>
|
|
15
|
+
<% end %>
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
<% content_for :title, "Snapshots" %>
|
|
2
|
+
|
|
3
|
+
<h2 class="rh-h2">Snapshots · <%= @snapshots.size %></h2>
|
|
4
|
+
<table class="rh-table">
|
|
5
|
+
<thead><tr><th>Snapshot</th><th>Queue</th><th class="r">Jobs</th><th class="r">Created</th><th class="r">Actions</th></tr></thead>
|
|
6
|
+
<tbody>
|
|
7
|
+
<% if @snapshots.empty? %>
|
|
8
|
+
<tr><td colspan="5" class="rh-empty">No snapshots yet — snapshot a queue before purging it.</td></tr>
|
|
9
|
+
<% else %>
|
|
10
|
+
<% @snapshots.each do |s| %>
|
|
11
|
+
<tr>
|
|
12
|
+
<td><span class="rh-sub"><%= s[:id] %></span></td>
|
|
13
|
+
<td><%= s[:queue] %></td>
|
|
14
|
+
<td class="r"><%= number_with_delimiter s[:count] %></td>
|
|
15
|
+
<td class="r"><span class="rh-sub"><%= s[:created_at] ? "#{time_ago_in_words(Time.at(s[:created_at]))} ago" : "—" %></span></td>
|
|
16
|
+
<td class="r">
|
|
17
|
+
<%= button_to "Restore", restore_snapshot_path(s[:id]), method: :post, class: "rh-btn", form_class: "rh-inline", data: { turbo_confirm: "Re-enqueue #{number_with_delimiter s[:count]} job(s) onto “#{s[:queue]}”?" } %>
|
|
18
|
+
<%= button_to "Delete", delete_snapshot_path(s[:id]), method: :post, class: "rh-btn rh-btn-danger", form_class: "rh-inline" %>
|
|
19
|
+
</td>
|
|
20
|
+
</tr>
|
|
21
|
+
<% end %>
|
|
22
|
+
<% end %>
|
|
23
|
+
</tbody>
|
|
24
|
+
</table>
|
|
25
|
+
<p class="rh-note">Snapshots are stored via the configured store (default: Redis). Restore re-enqueues onto the original queue.</p>
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
<% content_for :title, "Workers" %>
|
|
2
|
+
|
|
3
|
+
<h2 class="rh-h2">Workers · <%= @processes.size %> <%= "process".pluralize(@processes.size) %></h2>
|
|
4
|
+
<p class="rh-sub" style="margin:-4px 0 16px">
|
|
5
|
+
Fetch strategy:
|
|
6
|
+
<% if @fetch_active %>
|
|
7
|
+
<span class="rh-st rh-st-ok">RoundhouseUi::Fetch — pause-aware</span>
|
|
8
|
+
<% else %>
|
|
9
|
+
<span class="rh-st rh-st-paused">default — pause not enforced</span>
|
|
10
|
+
<% end %>
|
|
11
|
+
<span class="rh-sub" style="margin-left:6px">crash-reliable fetch (super_fetch) is Sidekiq Pro</span>
|
|
12
|
+
</p>
|
|
13
|
+
<table class="rh-table">
|
|
14
|
+
<thead><tr><th>Process</th><th class="r">Threads</th><th>Queues</th><th class="r">Started</th><th class="r">Heartbeat</th><th class="r">Actions</th></tr></thead>
|
|
15
|
+
<tbody>
|
|
16
|
+
<% if @processes.empty? %>
|
|
17
|
+
<tr><td colspan="6" class="rh-empty">No Sidekiq processes are running.</td></tr>
|
|
18
|
+
<% else %>
|
|
19
|
+
<% @processes.each do |p| %>
|
|
20
|
+
<% stale = p["beat"] && (Time.now.to_f - p["beat"].to_f) > 60 %>
|
|
21
|
+
<tr>
|
|
22
|
+
<td>
|
|
23
|
+
<%= p["hostname"] %>:<%= p["pid"] %><%
|
|
24
|
+
%><% if p.stopping? %> <span class="rh-err">· stopping</span><% elsif p["quiet"].to_s == "true" %> <span class="rh-sub">· quieting</span><% end %>
|
|
25
|
+
<br><span class="rh-sub"><%= [ p["tag"], ("v#{p["version"]}" if p["version"]), ("#{(p["rss"].to_i / 1024.0).round} MB" if p["rss"].to_i.positive?) ].compact.join(" · ") %></span>
|
|
26
|
+
</td>
|
|
27
|
+
<td class="r"><%= p["busy"] %>/<%= p["concurrency"] %></td>
|
|
28
|
+
<td><span class="rh-sub"><%= Array(p["queues"]).join(", ") %></span></td>
|
|
29
|
+
<td class="r"><span class="rh-sub"><%= p["started_at"] ? "#{time_ago_in_words(Time.at(p["started_at"]))} ago" : "—" %></span></td>
|
|
30
|
+
<td class="r <%= "rh-err" if stale %>"><%= p["beat"] ? "#{distance_of_time_in_words(Time.now, Time.at(p["beat"]))} ago" : "—" %></td>
|
|
31
|
+
<td class="r">
|
|
32
|
+
<%= button_to "Quiet", quiet_worker_path, method: :post, params: { identity: p.identity }, class: "rh-btn", form_class: "rh-inline" %>
|
|
33
|
+
<%= button_to "Stop", stop_worker_path, method: :post, params: { identity: p.identity }, class: "rh-btn rh-btn-danger", form_class: "rh-inline", data: { turbo_confirm: "Stop #{p["hostname"]}:#{p["pid"]}?" } %>
|
|
34
|
+
</td>
|
|
35
|
+
</tr>
|
|
36
|
+
<% end %>
|
|
37
|
+
<% end %>
|
|
38
|
+
</tbody>
|
|
39
|
+
</table>
|
data/config/routes.rb
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
RoundhouseUi::Engine.routes.draw do
|
|
2
|
+
root to: "dashboard#show"
|
|
3
|
+
get "stats" => "dashboard#stats", as: :dashboard_stats # JSON, polled for live updates
|
|
4
|
+
get "turbo.js" => "assets#turbo", as: :turbo_js # vendored Turbo, served same-origin
|
|
5
|
+
get "metrics" => "metrics#show", as: :metrics # derived metrics (separate from the live dashboard)
|
|
6
|
+
|
|
7
|
+
get "busy" => "busy#index", as: :busy
|
|
8
|
+
post "busy/:jid/cancel" => "busy#cancel", as: :cancel_job, constraints: { jid: /[^\/]+/ }
|
|
9
|
+
get "workers" => "workers#index", as: :workers
|
|
10
|
+
post "workers/quiet" => "workers#quiet", as: :quiet_worker
|
|
11
|
+
post "workers/stop" => "workers#stop", as: :stop_worker
|
|
12
|
+
|
|
13
|
+
get "queues" => "queues#index", as: :queues
|
|
14
|
+
# Queue names can contain dots/colons, so allow anything but a slash.
|
|
15
|
+
scope constraints: { name: /[^\/]+/ } do
|
|
16
|
+
post "queues/:name/purge" => "queues#purge", as: :purge_queue
|
|
17
|
+
post "queues/:name/pause" => "queues#pause", as: :pause_queue
|
|
18
|
+
post "queues/:name/resume" => "queues#resume", as: :resume_queue
|
|
19
|
+
post "queues/:name/snapshot" => "queues#snapshot", as: :snapshot_queue
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
get "snapshots" => "snapshots#index", as: :snapshots
|
|
23
|
+
scope constraints: { id: /[^\/]+/ } do
|
|
24
|
+
post "snapshots/:id/restore" => "snapshots#restore", as: :restore_snapshot
|
|
25
|
+
post "snapshots/:id/delete" => "snapshots#destroy", as: :delete_snapshot
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
get "scheduled" => "scheduled#index", as: :scheduled
|
|
29
|
+
post "scheduled/:jid/enqueue" => "scheduled#enqueue", as: :enqueue_scheduled
|
|
30
|
+
post "scheduled/:jid/delete" => "scheduled#destroy", as: :delete_scheduled
|
|
31
|
+
|
|
32
|
+
get "retries" => "retries#index", as: :retries
|
|
33
|
+
post "retries/:jid/run" => "retries#requeue", as: :run_retry
|
|
34
|
+
post "retries/:jid/delete" => "retries#destroy", as: :delete_retry
|
|
35
|
+
|
|
36
|
+
get "dead" => "dead#index", as: :dead_set
|
|
37
|
+
post "dead/bulk" => "dead#bulk", as: :bulk_dead
|
|
38
|
+
post "dead/:jid/retry" => "dead#requeue", as: :retry_dead_job
|
|
39
|
+
post "dead/:jid/delete" => "dead#destroy", as: :delete_dead_job
|
|
40
|
+
|
|
41
|
+
get "errors" => "errors#index", as: :errors
|
|
42
|
+
get "capsules" => "capsules#index", as: :capsules
|
|
43
|
+
get "redis" => "redis#show", as: :redis_info
|
|
44
|
+
get "audit" => "audit#index", as: :audit_log
|
|
45
|
+
|
|
46
|
+
# Job editing / enqueue (opt-in via RoundhouseUi.allow_job_editing)
|
|
47
|
+
get "jobs/new" => "jobs#new", as: :new_job
|
|
48
|
+
post "jobs" => "jobs#create", as: :jobs
|
|
49
|
+
scope constraints: { set: /dead|retry|scheduled/, jid: /[^\/]+/ } do
|
|
50
|
+
get "jobs/:set/:jid/edit" => "jobs#edit", as: :edit_job
|
|
51
|
+
get "jobs/:set/:jid" => "jobs#show" # read-only inspect (job_path)
|
|
52
|
+
post "jobs/:set/:jid" => "jobs#update", as: :job
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "sidekiq"
|
|
3
|
+
|
|
4
|
+
module RoundhouseUi
|
|
5
|
+
# An append-only audit trail of state-changing actions, kept in a capped Redis
|
|
6
|
+
# list. Answers "who purged that queue?" — the accountability Sidekiq Web lacks.
|
|
7
|
+
module Audit
|
|
8
|
+
KEY = "roundhouse:audit"
|
|
9
|
+
MAX = 1_000
|
|
10
|
+
|
|
11
|
+
module_function
|
|
12
|
+
|
|
13
|
+
def record(actor:, action:, target:)
|
|
14
|
+
entry = JSON.dump("actor" => actor.to_s, "action" => action.to_s, "target" => target.to_s, "at" => Time.now.to_f)
|
|
15
|
+
Sidekiq.redis do |conn|
|
|
16
|
+
conn.call("LPUSH", KEY, entry)
|
|
17
|
+
conn.call("LTRIM", KEY, 0, MAX - 1)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def recent(limit = 200)
|
|
22
|
+
Sidekiq.redis { |conn| conn.call("LRANGE", KEY, 0, limit - 1) }.map { |raw| JSON.parse(raw) }
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
require "roundhouse_ui/cancellation"
|
|
2
|
+
|
|
3
|
+
module RoundhouseUi
|
|
4
|
+
# Opt-in Sidekiq server middleware that drops a job whose JID was cancelled
|
|
5
|
+
# before it runs. Install it:
|
|
6
|
+
#
|
|
7
|
+
# Sidekiq.configure_server do |config|
|
|
8
|
+
# config.server_middleware { |chain| chain.add RoundhouseUi::CancelMiddleware }
|
|
9
|
+
# end
|
|
10
|
+
class CancelMiddleware
|
|
11
|
+
def call(_worker, job, _queue)
|
|
12
|
+
if RoundhouseUi::Cancellation.cancelled?(job["jid"])
|
|
13
|
+
RoundhouseUi::Cancellation.clear!(job["jid"])
|
|
14
|
+
return # acknowledge without running
|
|
15
|
+
end
|
|
16
|
+
yield
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
require "sidekiq"
|
|
2
|
+
|
|
3
|
+
module RoundhouseUi
|
|
4
|
+
# Cooperative job cancellation — pure OSS, no preemption (Ruby can't safely
|
|
5
|
+
# kill a running thread). Cancelled JIDs live in a Redis set:
|
|
6
|
+
#
|
|
7
|
+
# * RoundhouseUi::CancelMiddleware skips a job whose JID is cancelled when it
|
|
8
|
+
# is *about* to run (covers queued/scheduled/retry jobs).
|
|
9
|
+
# * A long-running job can call RoundhouseUi.cancelled?(jid) and bail out.
|
|
10
|
+
#
|
|
11
|
+
# The set expires so stale flags clean themselves up.
|
|
12
|
+
module Cancellation
|
|
13
|
+
KEY = "roundhouse:cancelled"
|
|
14
|
+
TTL = 86_400 # seconds
|
|
15
|
+
|
|
16
|
+
module_function
|
|
17
|
+
|
|
18
|
+
def cancel!(jid)
|
|
19
|
+
Sidekiq.redis do |conn|
|
|
20
|
+
conn.call("SADD", KEY, jid.to_s)
|
|
21
|
+
conn.call("EXPIRE", KEY, TTL)
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def cancelled?(jid)
|
|
26
|
+
Sidekiq.redis { |conn| conn.call("SISMEMBER", KEY, jid.to_s) } == 1
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def clear!(jid)
|
|
30
|
+
Sidekiq.redis { |conn| conn.call("SREM", KEY, jid.to_s) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def cancelled_jids
|
|
34
|
+
Sidekiq.redis { |conn| conn.call("SMEMBERS", KEY) }
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
require "sidekiq/fetch"
|
|
2
|
+
require "roundhouse_ui/pause"
|
|
3
|
+
|
|
4
|
+
module RoundhouseUi
|
|
5
|
+
# Opt-in fetch strategy that honors RoundhouseUi::Pause. Install it server-side:
|
|
6
|
+
#
|
|
7
|
+
# # config/initializers/sidekiq.rb
|
|
8
|
+
# Sidekiq.configure_server do |config|
|
|
9
|
+
# config[:fetch_class] = RoundhouseUi::Fetch
|
|
10
|
+
# end
|
|
11
|
+
#
|
|
12
|
+
# Until this fetcher is active, pausing a queue has no effect — the UI detects
|
|
13
|
+
# that (via the liveness beacon below) and warns rather than pretending.
|
|
14
|
+
#
|
|
15
|
+
# Inherits all of BasicFetch's behavior (weights, strict ordering, timeouts);
|
|
16
|
+
# we only filter the queue list. BasicFetch#retrieve_work already handles an
|
|
17
|
+
# empty list (sleep + return), so "all queues paused" is safe.
|
|
18
|
+
class Fetch < Sidekiq::BasicFetch
|
|
19
|
+
BEACON_INTERVAL = 5 # seconds; throttles the liveness write
|
|
20
|
+
|
|
21
|
+
def queues_cmd
|
|
22
|
+
touch_liveness
|
|
23
|
+
RoundhouseUi::Pause.reject_paused(super)
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
private
|
|
27
|
+
|
|
28
|
+
def touch_liveness
|
|
29
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
30
|
+
return if @rh_beat_at && (now - @rh_beat_at) < BEACON_INTERVAL
|
|
31
|
+
|
|
32
|
+
@rh_beat_at = now
|
|
33
|
+
RoundhouseUi::Pause.mark_fetch_alive!
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
require "sidekiq/api"
|
|
2
|
+
|
|
3
|
+
module RoundhouseUi
|
|
4
|
+
# Derived, single-snapshot metrics computed from data we already read
|
|
5
|
+
# (Sidekiq::Stats + the live ProcessSet). No storage and no deltas — these are
|
|
6
|
+
# instantaneous. Rate-based metrics (jobs/sec, velocity) are computed
|
|
7
|
+
# client-side from the dashboard's poll stream; per-class durations need the
|
|
8
|
+
# collector (a separate, opt-in piece).
|
|
9
|
+
class Metrics
|
|
10
|
+
def initialize(stats: Sidekiq::Stats.new, processes: Sidekiq::ProcessSet.new)
|
|
11
|
+
@stats = stats
|
|
12
|
+
@processes = processes
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Total worker threads across the live fleet.
|
|
16
|
+
def concurrency
|
|
17
|
+
@concurrency ||= @processes.sum { |process| process["concurrency"].to_i }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Worker threads currently running a job.
|
|
21
|
+
def busy
|
|
22
|
+
@stats.workers_size
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# Fraction of worker threads in use (0.0–1.0), or nil when no capacity is
|
|
26
|
+
# reporting in (no processes up) — so the view can show "—" instead of 0%.
|
|
27
|
+
def utilization
|
|
28
|
+
return nil if concurrency.zero?
|
|
29
|
+
|
|
30
|
+
busy.to_f / concurrency
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
# Idle worker threads.
|
|
34
|
+
def headroom
|
|
35
|
+
[ concurrency - busy, 0 ].max
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Everything waiting to run: live queues + scheduled + retrying.
|
|
39
|
+
def backlog
|
|
40
|
+
@stats.enqueued + @stats.scheduled_size + @stats.retry_size
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Share of processed jobs that failed, lifetime. Coarse (Sidekiq keeps only
|
|
44
|
+
# cumulative counters); the live failure rate is computed client-side.
|
|
45
|
+
def failure_ratio
|
|
46
|
+
return 0.0 if @stats.processed.zero?
|
|
47
|
+
|
|
48
|
+
@stats.failed.to_f / @stats.processed
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
require "cgi"
|
|
2
|
+
|
|
3
|
+
module RoundhouseUi
|
|
4
|
+
# Pluggable deep-links from a job to your APM/observability tool. The core
|
|
5
|
+
# never depends on Datadog (or anything) — it asks the configured adapter for
|
|
6
|
+
# a URL and renders a link only if one comes back.
|
|
7
|
+
#
|
|
8
|
+
# RoundhouseUi.observability = RoundhouseUi::Observability::DatadogAdapter.new(service: "trainual")
|
|
9
|
+
#
|
|
10
|
+
# Write your own (Honeycomb, Sentry, …) by duck-typing job_url/queue_url/label.
|
|
11
|
+
module Observability
|
|
12
|
+
# Default: no links anywhere.
|
|
13
|
+
class NullAdapter
|
|
14
|
+
def label = "trace"
|
|
15
|
+
def job_url(**) = nil
|
|
16
|
+
def queue_url(_name) = nil
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
class DatadogAdapter
|
|
20
|
+
def initialize(site: "datadoghq.com", service: nil, extra_query: nil)
|
|
21
|
+
@site = site
|
|
22
|
+
@service = service
|
|
23
|
+
@extra_query = extra_query
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def label = "Datadog"
|
|
27
|
+
|
|
28
|
+
def job_url(klass:, jid:, queue: nil)
|
|
29
|
+
terms = [ "@sidekiq.jid:#{jid}" ]
|
|
30
|
+
terms << "service:#{@service}" if @service
|
|
31
|
+
terms << @extra_query if @extra_query
|
|
32
|
+
traces_url(terms)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def queue_url(name)
|
|
36
|
+
traces_url([ "@sidekiq.queue:#{name}" ])
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
private
|
|
40
|
+
|
|
41
|
+
def traces_url(terms)
|
|
42
|
+
"https://app.#{@site}/apm/traces?query=#{CGI.escape(terms.compact.join(" "))}"
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|