dispatch_policy 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/MIT-LICENSE +16 -17
- data/README.md +449 -288
- data/app/assets/stylesheets/dispatch_policy/application.css +157 -0
- data/app/controllers/dispatch_policy/application_controller.rb +45 -1
- data/app/controllers/dispatch_policy/dashboard_controller.rb +91 -0
- data/app/controllers/dispatch_policy/partitions_controller.rb +122 -0
- data/app/controllers/dispatch_policy/policies_controller.rb +94 -241
- data/app/controllers/dispatch_policy/staged_jobs_controller.rb +9 -0
- data/app/models/dispatch_policy/adaptive_concurrency_stats.rb +11 -81
- data/app/models/dispatch_policy/inflight_job.rb +12 -0
- data/app/models/dispatch_policy/partition.rb +21 -0
- data/app/models/dispatch_policy/staged_job.rb +4 -97
- data/app/models/dispatch_policy/tick_sample.rb +11 -0
- data/app/views/dispatch_policy/dashboard/index.html.erb +109 -0
- data/app/views/dispatch_policy/partitions/index.html.erb +63 -0
- data/app/views/dispatch_policy/partitions/show.html.erb +106 -0
- data/app/views/dispatch_policy/policies/index.html.erb +15 -37
- data/app/views/dispatch_policy/policies/show.html.erb +140 -216
- data/app/views/dispatch_policy/shared/_capacity.html.erb +67 -0
- data/app/views/dispatch_policy/shared/_hints.html.erb +13 -0
- data/app/views/dispatch_policy/shared/_partition_row.html.erb +12 -0
- data/app/views/dispatch_policy/staged_jobs/show.html.erb +31 -0
- data/app/views/layouts/dispatch_policy/application.html.erb +95 -238
- data/config/routes.rb +18 -2
- data/db/migrate/20260501000001_create_dispatch_policy_tables.rb +103 -0
- data/lib/dispatch_policy/bypass.rb +23 -0
- data/lib/dispatch_policy/config.rb +85 -0
- data/lib/dispatch_policy/context.rb +50 -0
- data/lib/dispatch_policy/cursor_pagination.rb +121 -0
- data/lib/dispatch_policy/decision.rb +22 -0
- data/lib/dispatch_policy/engine.rb +4 -27
- data/lib/dispatch_policy/forwarder.rb +63 -0
- data/lib/dispatch_policy/gate.rb +10 -38
- data/lib/dispatch_policy/gates/adaptive_concurrency.rb +99 -97
- data/lib/dispatch_policy/gates/concurrency.rb +45 -26
- data/lib/dispatch_policy/gates/throttle.rb +65 -37
- data/lib/dispatch_policy/inflight_tracker.rb +174 -0
- data/lib/dispatch_policy/job_extension.rb +155 -0
- data/lib/dispatch_policy/operator_hints.rb +126 -0
- data/lib/dispatch_policy/pipeline.rb +48 -0
- data/lib/dispatch_policy/policy.rb +62 -47
- data/lib/dispatch_policy/policy_dsl.rb +120 -0
- data/lib/dispatch_policy/railtie.rb +35 -0
- data/lib/dispatch_policy/registry.rb +46 -0
- data/lib/dispatch_policy/repository.rb +723 -0
- data/lib/dispatch_policy/serializer.rb +36 -0
- data/lib/dispatch_policy/tick.rb +263 -172
- data/lib/dispatch_policy/tick_loop.rb +59 -26
- data/lib/dispatch_policy/version.rb +1 -1
- data/lib/dispatch_policy.rb +71 -46
- data/lib/generators/dispatch_policy/install/install_generator.rb +70 -0
- data/lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt +95 -0
- data/lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt +53 -0
- data/lib/generators/dispatch_policy/install/templates/initializer.rb.tt +11 -0
- metadata +101 -43
- data/CHANGELOG.md +0 -12
- data/app/models/dispatch_policy/partition_inflight_count.rb +0 -42
- data/app/models/dispatch_policy/partition_observation.rb +0 -49
- data/app/models/dispatch_policy/throttle_bucket.rb +0 -41
- data/db/migrate/20260424000001_create_dispatch_policy_tables.rb +0 -80
- data/db/migrate/20260424000002_create_adaptive_concurrency_stats.rb +0 -22
- data/db/migrate/20260424000003_create_adaptive_concurrency_samples.rb +0 -25
- data/db/migrate/20260424000004_rename_samples_to_partition_observations.rb +0 -32
- data/lib/dispatch_policy/active_job_perform_all_later_patch.rb +0 -32
- data/lib/dispatch_policy/dispatch_context.rb +0 -53
- data/lib/dispatch_policy/dispatchable.rb +0 -120
- data/lib/dispatch_policy/gates/fair_interleave.rb +0 -32
- data/lib/dispatch_policy/gates/global_cap.rb +0 -26
- data/lib/dispatch_policy/install_generator.rb +0 -23
|
@@ -1,241 +1,165 @@
|
|
|
1
|
-
<
|
|
2
|
-
<h1><%= @policy_name %> <small class="muted"><%= @job_class.name %></small></h1>
|
|
1
|
+
<h1>Policy <code><%= @policy_name %></code></h1>
|
|
3
2
|
|
|
4
|
-
<
|
|
5
|
-
<div class="
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
</div>
|
|
10
|
-
<div class="summary-item">
|
|
11
|
-
<div class="label">Admitted</div>
|
|
12
|
-
<div class="value"><%= @admitted_count %></div>
|
|
13
|
-
</div>
|
|
14
|
-
<div class="summary-item">
|
|
15
|
-
<div class="label">Completed (24h)</div>
|
|
16
|
-
<div class="value"><%= @completed_24h %></div>
|
|
17
|
-
</div>
|
|
18
|
-
<div class="summary-item">
|
|
19
|
-
<div class="label">Gates</div>
|
|
20
|
-
<div class="value" style="font-size: 1em;">
|
|
21
|
-
<% (@policy&.gates || []).each do |g| %>
|
|
22
|
-
<span class="badge"><%= g.name %></span>
|
|
23
|
-
<% end %>
|
|
24
|
-
</div>
|
|
25
|
-
</div>
|
|
26
|
-
</div>
|
|
27
|
-
|
|
28
|
-
<% if @adaptive_global.any? { |v| !v.nil? } %>
|
|
29
|
-
<h2>Adaptive — EWMA queue lag (avg across partitions, last hour)</h2>
|
|
30
|
-
<div class="chart-global">
|
|
31
|
-
<canvas data-chart="line"
|
|
32
|
-
data-label="avg ewma_latency_ms"
|
|
33
|
-
data-labels='<%= @chart_labels.to_json %>'
|
|
34
|
-
data-values='<%= @adaptive_global.to_json %>'
|
|
35
|
-
data-counts='<%= @completions_global.to_json %>'></canvas>
|
|
36
|
-
</div>
|
|
37
|
-
<% end %>
|
|
3
|
+
<section class="dp-stats">
|
|
4
|
+
<div class="dp-stat"><span class="dp-stat-label">Partitions</span><span class="dp-stat-value"><%= format_count(@totals[:partitions]) %></span></div>
|
|
5
|
+
<div class="dp-stat"><span class="dp-stat-label">Pending</span><span class="dp-stat-value"><%= format_count(@totals[:pending]) %></span></div>
|
|
6
|
+
<div class="dp-stat"><span class="dp-stat-label">In flight</span><span class="dp-stat-value"><%= format_count(@totals[:in_flight]) %></span></div>
|
|
7
|
+
</section>
|
|
38
8
|
|
|
39
|
-
|
|
9
|
+
<%= render "dispatch_policy/shared/hints", hints: @hints %>
|
|
10
|
+
<%= render "dispatch_policy/shared/capacity",
|
|
11
|
+
capacity: @capacity,
|
|
12
|
+
pending_trend: @pending_trend,
|
|
13
|
+
pending_buckets: @sparkline %>
|
|
40
14
|
|
|
41
|
-
|
|
42
|
-
<
|
|
43
|
-
|
|
44
|
-
<table data-watched-table data-policy-name="<%= @policy_name %>">
|
|
15
|
+
<section class="dp-section">
|
|
16
|
+
<h2>Throughput</h2>
|
|
17
|
+
<table class="dp-table">
|
|
45
18
|
<thead>
|
|
46
|
-
<tr>
|
|
47
|
-
<th></th>
|
|
48
|
-
<th>Source</th>
|
|
49
|
-
<th>Partition</th>
|
|
50
|
-
<th class="text-end">Pending (eligible)</th>
|
|
51
|
-
<th class="text-end">Pending (scheduled)</th>
|
|
52
|
-
<th class="text-end">In-flight</th>
|
|
53
|
-
<th class="text-end">Completed (24h)</th>
|
|
54
|
-
<th class="text-end">Adaptive cap</th>
|
|
55
|
-
<th class="text-end">EWMA latency (ms)</th>
|
|
56
|
-
</tr>
|
|
19
|
+
<tr><th>Window</th><th class="dp-num">Admitted</th><th class="dp-num">Ticks</th><th class="dp-num">Avg tick</th><th class="dp-num">Max tick</th><th class="dp-num">Forward fail</th><th class="dp-num">Partitions seen</th></tr>
|
|
57
20
|
</thead>
|
|
58
21
|
<tbody>
|
|
59
|
-
<% @
|
|
60
|
-
|
|
61
|
-
<td><button type="button" class="chip" data-unwatch="<%= row[:partition] %>" title="Stop watching">×</button></td>
|
|
62
|
-
<td><%= row[:source] %></td>
|
|
63
|
-
<td><code><%= row[:partition] %></code></td>
|
|
64
|
-
<td class="text-end"><%= row[:eligible] %></td>
|
|
65
|
-
<td class="text-end"><%= row[:scheduled] %></td>
|
|
66
|
-
<td class="text-end"><%= row[:in_flight] %></td>
|
|
67
|
-
<td class="text-end"><%= row[:completed_24h] %></td>
|
|
68
|
-
<td class="text-end">
|
|
69
|
-
<% if row[:current_max] %><strong><%= row[:current_max] %></strong><% else %><span class="muted">—</span><% end %>
|
|
70
|
-
</td>
|
|
71
|
-
<td class="text-end">
|
|
72
|
-
<% if row[:ewma_latency_ms] %><%= row[:ewma_latency_ms] %><% else %><span class="muted">—</span><% end %>
|
|
73
|
-
</td>
|
|
74
|
-
</tr>
|
|
75
|
-
<%
|
|
76
|
-
series = @adaptive_samples[row[:partition]] || Array.new(@chart_slots.size)
|
|
77
|
-
counts = @completions_samples[row[:partition]] || Array.new(@chart_slots.size, 0)
|
|
78
|
-
has_samples = series.any? { |v| !v.nil? } || counts.any? { |c| c.positive? }
|
|
79
|
-
%>
|
|
22
|
+
<% @windows.each do |label, m| %>
|
|
23
|
+
<% fail_pct = m[:jobs_admitted].positive? ? (m[:forward_failures].to_f / m[:jobs_admitted] * 100) : 0.0 %>
|
|
80
24
|
<tr>
|
|
81
|
-
<td
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
<div class="sparkline-wrap sparkline-empty"><span class="muted">no completions in the last hour yet</span></div>
|
|
25
|
+
<td><strong><%= label %></strong></td>
|
|
26
|
+
<td class="dp-num"><%= format_count(m[:jobs_admitted]) %></td>
|
|
27
|
+
<td class="dp-num"><%= format_count(m[:ticks]) %></td>
|
|
28
|
+
<td class="dp-num"><%= format_duration_ms(m[:avg_duration_ms]) %></td>
|
|
29
|
+
<td class="dp-num"><%= format_duration_ms(m[:max_duration_ms]) %></td>
|
|
30
|
+
<td class="dp-num <%= "dp-warn" if fail_pct >= 1.0 %>">
|
|
31
|
+
<%= format_count(m[:forward_failures]) %>
|
|
32
|
+
<% if m[:jobs_admitted].positive? && m[:forward_failures].positive? %>
|
|
33
|
+
<span style="color:#9ca3af; font-size:11px;">(<%= format("%.2f%%", fail_pct) %>)</span>
|
|
91
34
|
<% end %>
|
|
92
35
|
</td>
|
|
36
|
+
<td class="dp-num"><%= format_count(m[:partitions_seen]) %></td>
|
|
93
37
|
</tr>
|
|
94
38
|
<% end %>
|
|
95
39
|
</tbody>
|
|
96
40
|
</table>
|
|
97
|
-
<% end %>
|
|
98
41
|
|
|
99
|
-
|
|
42
|
+
<% spark_admit = @sparkline.map { |b| b[:jobs_admitted] } %>
|
|
43
|
+
<% if spark_admit.any? %>
|
|
44
|
+
<p class="dp-spark">
|
|
45
|
+
Last 30 minutes (1-minute buckets): <code><%= sparkline(spark_admit, width: 30) %></code>
|
|
46
|
+
peak: <%= format_count(spark_admit.max) %> jobs/min
|
|
47
|
+
</p>
|
|
48
|
+
<% end %>
|
|
49
|
+
</section>
|
|
100
50
|
|
|
101
|
-
<
|
|
102
|
-
|
|
103
|
-
|
|
51
|
+
<section class="dp-section">
|
|
52
|
+
<h2>Round-trip</h2>
|
|
53
|
+
<% if @round_trip[:active_partitions].zero? %>
|
|
54
|
+
<p class="dp-empty">No active partitions with pending jobs in this policy.</p>
|
|
104
55
|
<% else %>
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
56
|
+
<div class="dp-stats">
|
|
57
|
+
<div class="dp-stat"><span class="dp-stat-label">Active w/ pending</span><span class="dp-stat-value"><%= format_count(@round_trip[:active_partitions]) %></span></div>
|
|
58
|
+
<div class="dp-stat"><span class="dp-stat-label">Never checked</span><span class="dp-stat-value <%= "dp-warn" if @round_trip[:never_checked].positive? %>"><%= format_count(@round_trip[:never_checked]) %></span></div>
|
|
59
|
+
<div class="dp-stat"><span class="dp-stat-label">In backoff</span><span class="dp-stat-value"><%= format_count(@round_trip[:in_backoff]) %></span></div>
|
|
60
|
+
<div class="dp-stat"><span class="dp-stat-label">Oldest age</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:oldest_age_seconds]) %></span></div>
|
|
61
|
+
<div class="dp-stat"><span class="dp-stat-label">P50</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:p50_age_seconds]) %></span></div>
|
|
62
|
+
<div class="dp-stat"><span class="dp-stat-label">P95</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:p95_age_seconds]) %></span></div>
|
|
63
|
+
</div>
|
|
64
|
+
<% end %>
|
|
65
|
+
</section>
|
|
66
|
+
|
|
67
|
+
<section class="dp-section">
|
|
68
|
+
<h2>Denials (last 15m)</h2>
|
|
69
|
+
<% if @denied_reasons.empty? %>
|
|
70
|
+
<p class="dp-empty">No denials recorded in the window.</p>
|
|
71
|
+
<% else %>
|
|
72
|
+
<table class="dp-table">
|
|
73
|
+
<thead><tr><th>Reason</th><th class="dp-num">Count</th></tr></thead>
|
|
74
|
+
<tbody>
|
|
75
|
+
<% @denied_reasons.each do |reason, count| %>
|
|
76
|
+
<tr><td><code><%= reason %></code></td><td class="dp-num"><%= format_count(count) %></td></tr>
|
|
77
|
+
<% end %>
|
|
78
|
+
</tbody>
|
|
79
|
+
</table>
|
|
80
|
+
<p class="dp-hint">
|
|
81
|
+
<code>throttle_empty</code> dominating? Raise the policy's <code>rate</code> or accept the throttle and shard tenants.
|
|
82
|
+
<code>concurrency_full</code> dominating? Raise <code>max</code> on the concurrency gate or scale the worker pool.
|
|
83
|
+
</p>
|
|
84
|
+
<% end %>
|
|
85
|
+
</section>
|
|
86
|
+
|
|
87
|
+
<section class="dp-section">
|
|
88
|
+
<h2>Gates</h2>
|
|
89
|
+
<% if @policy_object %>
|
|
90
|
+
<ul class="dp-list">
|
|
91
|
+
<% @policy_object.gates.each do |gate| %>
|
|
92
|
+
<li><strong><%= gate.name %></strong> (<%= gate.class.name.demodulize %>)</li>
|
|
113
93
|
<% end %>
|
|
114
|
-
|
|
94
|
+
</ul>
|
|
95
|
+
<p>Retry strategy: <code><%= @policy_object.retry_strategy %></code></p>
|
|
96
|
+
<% else %>
|
|
97
|
+
<p class="dp-empty">Policy <code><%= @policy_name %></code> is not registered in this process.</p>
|
|
98
|
+
<% end %>
|
|
99
|
+
</section>
|
|
115
100
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
arrow = active ? (@partition_dir == "asc" ? " ▲" : " ▼") : ""
|
|
124
|
-
content_tag(:th, class: (align_end ? "text-end" : nil)) do
|
|
125
|
-
link_to "#{label}#{arrow}",
|
|
126
|
-
url_for(sort: col, dir: next_dir, q: @partition_search.presence, page: nil, watch: params[:watch]),
|
|
127
|
-
class: (active ? "sorted" : nil)
|
|
128
|
-
end
|
|
129
|
-
}
|
|
130
|
-
%>
|
|
131
|
-
<table data-policy-name="<%= @policy_name %>">
|
|
132
|
-
<thead>
|
|
101
|
+
<section class="dp-section">
|
|
102
|
+
<h2>Top partitions by lifetime admitted</h2>
|
|
103
|
+
<% if @top_admitted.any? %>
|
|
104
|
+
<table class="dp-table">
|
|
105
|
+
<thead><tr><th>Partition key</th><th class="dp-num">Lifetime admitted</th><th class="dp-num">Pending</th><th>Last admit</th></tr></thead>
|
|
106
|
+
<tbody>
|
|
107
|
+
<% @top_admitted.each do |p| %>
|
|
133
108
|
<tr>
|
|
134
|
-
<
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
<%= sort_link.call("In-flight", "in_flight", align_end: true) %>
|
|
139
|
-
<%= sort_link.call("Completed (24h)", "completed_24h", align_end: true) %>
|
|
140
|
-
<%= sort_link.call("Last enqueue", "last_enqueued_at") %>
|
|
141
|
-
<%= sort_link.call("Last dispatch", "last_dispatched_at") %>
|
|
109
|
+
<td><%= link_to p.partition_key, partition_path(p), class: "dp-link" %></td>
|
|
110
|
+
<td class="dp-num"><%= format_count(p.total_admitted) %></td>
|
|
111
|
+
<td class="dp-num"><%= format_count(p.pending_count) %></td>
|
|
112
|
+
<td><%= format_time(p.last_admit_at) %></td>
|
|
142
113
|
</tr>
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
<td>
|
|
149
|
-
<% if @watched_keys.include?(row[:partition]) %>
|
|
150
|
-
<span class="chip on" title="Watched">✓</span>
|
|
151
|
-
<% else %>
|
|
152
|
-
<button type="button" class="chip" data-watch="<%= row[:partition] %>" data-turbo-frame="_top" title="Watch this partition">+</button>
|
|
153
|
-
<% end %>
|
|
154
|
-
</td>
|
|
155
|
-
<td><%= row[:source] %></td>
|
|
156
|
-
<td><code><%= row[:partition] %></code></td>
|
|
157
|
-
<td class="text-end"><%= pending_total %></td>
|
|
158
|
-
<td class="text-end"><%= row[:in_flight] %></td>
|
|
159
|
-
<td class="text-end"><%= row[:completed_24h] %></td>
|
|
160
|
-
<td>
|
|
161
|
-
<% if row[:last_enqueued_at] %>
|
|
162
|
-
<span title="<%= row[:last_enqueued_at] %>"><%= time_ago_in_words(row[:last_enqueued_at]) %> ago</span>
|
|
163
|
-
<% else %>
|
|
164
|
-
<span class="muted">—</span>
|
|
165
|
-
<% end %>
|
|
166
|
-
</td>
|
|
167
|
-
<td>
|
|
168
|
-
<% if row[:last_dispatched_at] %>
|
|
169
|
-
<span title="<%= row[:last_dispatched_at] %>"><%= time_ago_in_words(row[:last_dispatched_at]) %> ago</span>
|
|
170
|
-
<% else %>
|
|
171
|
-
<span class="muted">—</span>
|
|
172
|
-
<% end %>
|
|
173
|
-
</td>
|
|
174
|
-
</tr>
|
|
175
|
-
<% end %>
|
|
176
|
-
</tbody>
|
|
177
|
-
</table>
|
|
178
|
-
|
|
179
|
-
<% total_pages = (@partition_total_list / DispatchPolicy::PoliciesController::PARTITION_LIST_PAGE_SIZE.to_f).ceil %>
|
|
180
|
-
<% if total_pages > 1 %>
|
|
181
|
-
<% page_params = { q: @partition_search.presence, watch: params[:watch], sort: params[:sort], dir: params[:dir] } %>
|
|
182
|
-
<div class="pagination">
|
|
183
|
-
<% if @partition_page > 1 %>
|
|
184
|
-
<%= link_to "← prev", url_for(page_params.merge(page: @partition_page - 1)) %>
|
|
185
|
-
<% end %>
|
|
186
|
-
<span class="muted">page <%= @partition_page %> / <%= total_pages %></span>
|
|
187
|
-
<% if @partition_page < total_pages %>
|
|
188
|
-
<%= link_to "next →", url_for(page_params.merge(page: @partition_page + 1)) %>
|
|
189
|
-
<% end %>
|
|
190
|
-
</div>
|
|
191
|
-
<% end %>
|
|
192
|
-
<% end %>
|
|
114
|
+
<% end %>
|
|
115
|
+
</tbody>
|
|
116
|
+
</table>
|
|
117
|
+
<% else %>
|
|
118
|
+
<p class="dp-empty">No partitions have admitted any jobs yet.</p>
|
|
193
119
|
<% end %>
|
|
194
|
-
</
|
|
120
|
+
</section>
|
|
195
121
|
|
|
196
|
-
<
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
<thead>
|
|
202
|
-
<tr><th>Gate</th><th>Partition</th><th>Tokens</th><th>Refilled at</th></tr>
|
|
203
|
-
</thead>
|
|
204
|
-
<tbody>
|
|
205
|
-
<% @throttle_buckets.each do |b| %>
|
|
122
|
+
<section class="dp-section">
|
|
123
|
+
<h2>Top partitions by pending</h2>
|
|
124
|
+
<% if @partitions.any? %>
|
|
125
|
+
<table class="dp-table">
|
|
126
|
+
<thead>
|
|
206
127
|
<tr>
|
|
207
|
-
<
|
|
208
|
-
<
|
|
209
|
-
<
|
|
210
|
-
<td><%= b.refilled_at.to_s(:short) rescue b.refilled_at %></td>
|
|
128
|
+
<th>Partition key</th><th>Status</th>
|
|
129
|
+
<th class="dp-num">Pending</th>
|
|
130
|
+
<th>Next eligible</th><th>Last admit</th><th></th>
|
|
211
131
|
</tr>
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
132
|
+
</thead>
|
|
133
|
+
<tbody>
|
|
134
|
+
<% @partitions.each do |p| %>
|
|
135
|
+
<tr>
|
|
136
|
+
<td><%= link_to p.partition_key, partition_path(p), class: "dp-link" %></td>
|
|
137
|
+
<td><%= p.status %></td>
|
|
138
|
+
<td class="dp-num"><%= format_count(p.pending_count) %></td>
|
|
139
|
+
<td><%= format_time(p.next_eligible_at) %></td>
|
|
140
|
+
<td><%= format_time(p.last_admit_at) %></td>
|
|
141
|
+
<td><%= link_to "→", partition_path(p), class: "dp-link" %></td>
|
|
142
|
+
</tr>
|
|
143
|
+
<% end %>
|
|
144
|
+
</tbody>
|
|
145
|
+
</table>
|
|
146
|
+
<% else %>
|
|
147
|
+
<p class="dp-empty">No partitions for this policy yet.</p>
|
|
148
|
+
<% end %>
|
|
149
|
+
</section>
|
|
216
150
|
|
|
217
|
-
<
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
</
|
|
228
|
-
<
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
<td><%= job.dedupe_key ? content_tag(:code, job.dedupe_key) : content_tag(:span, '—', class: 'muted') %></td>
|
|
233
|
-
<td><%= job.round_robin_key ? content_tag(:code, job.round_robin_key) : content_tag(:span, '—', class: 'muted') %></td>
|
|
234
|
-
<td><%= job.priority %></td>
|
|
235
|
-
<td><%= job.staged_at %></td>
|
|
236
|
-
<td><%= job.not_before_at || content_tag(:span, '—', class: 'muted') %></td>
|
|
237
|
-
</tr>
|
|
238
|
-
<% end %>
|
|
239
|
-
</tbody>
|
|
240
|
-
</table>
|
|
241
|
-
<% end %>
|
|
151
|
+
<section class="dp-section">
|
|
152
|
+
<h2>Actions</h2>
|
|
153
|
+
<%= button_to "Pause all partitions", pause_policy_path(@policy_name), class: "dp-btn", method: :post, form: { class: "dp-form-inline" } %>
|
|
154
|
+
<%= button_to "Resume all partitions", resume_policy_path(@policy_name), class: "dp-btn dp-btn-ok", method: :post, form: { class: "dp-form-inline" } %>
|
|
155
|
+
<%= button_to "Drain policy", drain_policy_path(@policy_name),
|
|
156
|
+
class: "dp-btn dp-btn-warn",
|
|
157
|
+
method: :post,
|
|
158
|
+
form: { class: "dp-form-inline",
|
|
159
|
+
onsubmit: "return confirm('Force-admit every staged job across every partition of this policy, bypassing all gates?');" } %>
|
|
160
|
+
<p class="dp-hint">
|
|
161
|
+
<strong>Pause</strong> stops admission but keeps staging — the queue keeps filling, in-flight jobs finish.
|
|
162
|
+
<strong>Drain</strong> empties the staging table by force-admitting every job (bypassing gates).
|
|
163
|
+
Capped at 10,000 jobs per click — click again for more.
|
|
164
|
+
</p>
|
|
165
|
+
</section>
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
<%
|
|
2
|
+
capacity = local_assigns.fetch(:capacity)
|
|
3
|
+
pending_trend = local_assigns.fetch(:pending_trend, :flat)
|
|
4
|
+
pending_buckets = local_assigns[:pending_buckets]
|
|
5
|
+
|
|
6
|
+
jps = capacity[:admitted_per_second] || (capacity[:admitted_per_minute].to_f / 60.0)
|
|
7
|
+
target = capacity[:adapter_target_jps]
|
|
8
|
+
ratio_jps = (target && target.positive?) ? jps / target : nil
|
|
9
|
+
tick_ratio = (capacity[:tick_max_duration_ms].to_i.positive? ?
|
|
10
|
+
capacity[:avg_tick_ms].to_f / capacity[:tick_max_duration_ms] : nil)
|
|
11
|
+
|
|
12
|
+
arrow = case pending_trend
|
|
13
|
+
when :up then "↑"
|
|
14
|
+
when :down then "↓"
|
|
15
|
+
else "→"
|
|
16
|
+
end
|
|
17
|
+
%>
|
|
18
|
+
|
|
19
|
+
<section class="dp-section">
|
|
20
|
+
<h2>Capacity headroom</h2>
|
|
21
|
+
<div class="dp-stats">
|
|
22
|
+
<div class="dp-stat">
|
|
23
|
+
<span class="dp-stat-label">Admit rate (1m)</span>
|
|
24
|
+
<span class="dp-stat-value"><%= format("%.1f", jps) %> jobs/sec</span>
|
|
25
|
+
</div>
|
|
26
|
+
<% if ratio_jps %>
|
|
27
|
+
<div class="dp-stat">
|
|
28
|
+
<span class="dp-stat-label">Of adapter ceiling (<%= target %>/sec)</span>
|
|
29
|
+
<span class="dp-stat-value <%= "dp-warn" if ratio_jps >= 0.7 %>">
|
|
30
|
+
<%= format("%.0f%%", ratio_jps * 100) %>
|
|
31
|
+
</span>
|
|
32
|
+
</div>
|
|
33
|
+
<% end %>
|
|
34
|
+
<div class="dp-stat">
|
|
35
|
+
<span class="dp-stat-label">Avg tick</span>
|
|
36
|
+
<span class="dp-stat-value"><%= format_duration_ms(capacity[:avg_tick_ms]) %></span>
|
|
37
|
+
</div>
|
|
38
|
+
<% if tick_ratio %>
|
|
39
|
+
<div class="dp-stat">
|
|
40
|
+
<span class="dp-stat-label">Of tick_max_duration</span>
|
|
41
|
+
<span class="dp-stat-value <%= "dp-warn" if tick_ratio >= 0.6 %>">
|
|
42
|
+
<%= format("%.0f%%", tick_ratio * 100) %>
|
|
43
|
+
</span>
|
|
44
|
+
</div>
|
|
45
|
+
<% end %>
|
|
46
|
+
<div class="dp-stat">
|
|
47
|
+
<span class="dp-stat-label">Pending trend (30m)</span>
|
|
48
|
+
<span class="dp-stat-value <%= "dp-warn" if pending_trend == :up %>">
|
|
49
|
+
<%= arrow %> <%= pending_trend %>
|
|
50
|
+
</span>
|
|
51
|
+
</div>
|
|
52
|
+
</div>
|
|
53
|
+
|
|
54
|
+
<% if pending_buckets && pending_buckets.any? %>
|
|
55
|
+
<p class="dp-spark">
|
|
56
|
+
Pending across the last 30 min:
|
|
57
|
+
<code><%= sparkline(pending_buckets.map { |b| b[:pending_total] }, width: 30) %></code>
|
|
58
|
+
</p>
|
|
59
|
+
<% end %>
|
|
60
|
+
<% if target.nil? %>
|
|
61
|
+
<p class="dp-hint">
|
|
62
|
+
Set <code>DispatchPolicy.config.adapter_throughput_target = 3500</code>
|
|
63
|
+
(jobs/sec) in your initializer to see the admit rate as a percentage.
|
|
64
|
+
Locally we measure ~3500/sec per worker against good_job; calibrate to your prod adapter.
|
|
65
|
+
</p>
|
|
66
|
+
<% end %>
|
|
67
|
+
</section>
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
<% if hints.any? %>
|
|
2
|
+
<section class="dp-section">
|
|
3
|
+
<h2>Hints</h2>
|
|
4
|
+
<ul class="dp-list dp-hint-list">
|
|
5
|
+
<% hints.each do |hint| %>
|
|
6
|
+
<li class="dp-hint-<%= hint.level %>">
|
|
7
|
+
<span class="dp-hint-badge"><%= hint.level.to_s.upcase %></span>
|
|
8
|
+
<%= hint.message %>
|
|
9
|
+
</li>
|
|
10
|
+
<% end %>
|
|
11
|
+
</ul>
|
|
12
|
+
</section>
|
|
13
|
+
<% end %>
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
<tr id="partition_<%= partition.id %>" class="<%= "dp-row-paused" if partition.paused? %>">
|
|
2
|
+
<td><%= link_to partition.policy_name, policy_path(partition.policy_name), class: "dp-link" %></td>
|
|
3
|
+
<td><code><%= partition.shard %></code></td>
|
|
4
|
+
<td><%= partition.queue_name || "—" %></td>
|
|
5
|
+
<td><%= link_to partition.partition_key, partition_path(partition), class: "dp-link" %></td>
|
|
6
|
+
<td><%= partition.status %></td>
|
|
7
|
+
<td class="dp-num"><%= format_count(partition.pending_count) %></td>
|
|
8
|
+
<td class="dp-num"><%= format_count(partition.total_admitted) %></td>
|
|
9
|
+
<td><%= format_time(partition.next_eligible_at) %></td>
|
|
10
|
+
<td><%= format_time(partition.last_admit_at) %></td>
|
|
11
|
+
<td><%= format_time(partition.last_enqueued_at) %></td>
|
|
12
|
+
</tr>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
<h1>Staged job #<%= @job.id %></h1>
|
|
2
|
+
|
|
3
|
+
<section class="dp-stats">
|
|
4
|
+
<div class="dp-stat"><span class="dp-stat-label">Policy</span><span class="dp-stat-value"><%= @job.policy_name %></span></div>
|
|
5
|
+
<div class="dp-stat"><span class="dp-stat-label">Class</span><span class="dp-stat-value"><code><%= @job.job_class %></code></span></div>
|
|
6
|
+
<div class="dp-stat"><span class="dp-stat-label">Queue</span><span class="dp-stat-value"><%= @job.queue_name || "—" %></span></div>
|
|
7
|
+
<div class="dp-stat"><span class="dp-stat-label">Priority</span><span class="dp-stat-value"><%= @job.priority %></span></div>
|
|
8
|
+
</section>
|
|
9
|
+
|
|
10
|
+
<section class="dp-section">
|
|
11
|
+
<h2>Partition</h2>
|
|
12
|
+
<p><code><%= @job.partition_key %></code></p>
|
|
13
|
+
</section>
|
|
14
|
+
|
|
15
|
+
<section class="dp-section">
|
|
16
|
+
<h2>Timing</h2>
|
|
17
|
+
<ul class="dp-list">
|
|
18
|
+
<li>Enqueued: <%= format_time(@job.enqueued_at) %></li>
|
|
19
|
+
<li>Scheduled: <%= format_time(@job.scheduled_at) %></li>
|
|
20
|
+
</ul>
|
|
21
|
+
</section>
|
|
22
|
+
|
|
23
|
+
<section class="dp-section">
|
|
24
|
+
<h2>Context snapshot</h2>
|
|
25
|
+
<pre class="dp-json"><%= JSON.pretty_generate(@job.context || {}) %></pre>
|
|
26
|
+
</section>
|
|
27
|
+
|
|
28
|
+
<section class="dp-section">
|
|
29
|
+
<h2>ActiveJob payload</h2>
|
|
30
|
+
<pre class="dp-json"><%= JSON.pretty_generate(@job.job_data || {}) %></pre>
|
|
31
|
+
</section>
|