dispatch_policy 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/MIT-LICENSE +16 -17
- data/README.md +433 -388
- data/app/assets/stylesheets/dispatch_policy/application.css +157 -0
- data/app/controllers/dispatch_policy/application_controller.rb +45 -1
- data/app/controllers/dispatch_policy/dashboard_controller.rb +91 -0
- data/app/controllers/dispatch_policy/partitions_controller.rb +122 -0
- data/app/controllers/dispatch_policy/policies_controller.rb +94 -267
- data/app/controllers/dispatch_policy/staged_jobs_controller.rb +9 -0
- data/app/models/dispatch_policy/adaptive_concurrency_stats.rb +11 -81
- data/app/models/dispatch_policy/inflight_job.rb +12 -0
- data/app/models/dispatch_policy/partition.rb +21 -0
- data/app/models/dispatch_policy/staged_job.rb +4 -97
- data/app/models/dispatch_policy/tick_sample.rb +11 -0
- data/app/views/dispatch_policy/dashboard/index.html.erb +109 -0
- data/app/views/dispatch_policy/partitions/index.html.erb +63 -0
- data/app/views/dispatch_policy/partitions/show.html.erb +106 -0
- data/app/views/dispatch_policy/policies/index.html.erb +15 -37
- data/app/views/dispatch_policy/policies/show.html.erb +139 -223
- data/app/views/dispatch_policy/shared/_capacity.html.erb +67 -0
- data/app/views/dispatch_policy/shared/_hints.html.erb +13 -0
- data/app/views/dispatch_policy/shared/_partition_row.html.erb +12 -0
- data/app/views/dispatch_policy/staged_jobs/show.html.erb +31 -0
- data/app/views/layouts/dispatch_policy/application.html.erb +95 -238
- data/config/routes.rb +18 -2
- data/db/migrate/20260501000001_create_dispatch_policy_tables.rb +103 -0
- data/lib/dispatch_policy/bypass.rb +23 -0
- data/lib/dispatch_policy/config.rb +85 -0
- data/lib/dispatch_policy/context.rb +50 -0
- data/lib/dispatch_policy/cursor_pagination.rb +121 -0
- data/lib/dispatch_policy/decision.rb +22 -0
- data/lib/dispatch_policy/engine.rb +4 -27
- data/lib/dispatch_policy/forwarder.rb +63 -0
- data/lib/dispatch_policy/gate.rb +10 -38
- data/lib/dispatch_policy/gates/adaptive_concurrency.rb +99 -97
- data/lib/dispatch_policy/gates/concurrency.rb +45 -26
- data/lib/dispatch_policy/gates/throttle.rb +65 -41
- data/lib/dispatch_policy/inflight_tracker.rb +174 -0
- data/lib/dispatch_policy/job_extension.rb +155 -0
- data/lib/dispatch_policy/operator_hints.rb +126 -0
- data/lib/dispatch_policy/pipeline.rb +48 -0
- data/lib/dispatch_policy/policy.rb +61 -59
- data/lib/dispatch_policy/policy_dsl.rb +120 -0
- data/lib/dispatch_policy/railtie.rb +35 -0
- data/lib/dispatch_policy/registry.rb +46 -0
- data/lib/dispatch_policy/repository.rb +723 -0
- data/lib/dispatch_policy/serializer.rb +36 -0
- data/lib/dispatch_policy/tick.rb +260 -256
- data/lib/dispatch_policy/tick_loop.rb +59 -26
- data/lib/dispatch_policy/version.rb +1 -1
- data/lib/dispatch_policy.rb +71 -52
- data/lib/generators/dispatch_policy/install/install_generator.rb +70 -0
- data/lib/generators/dispatch_policy/install/templates/create_dispatch_policy_tables.rb.tt +95 -0
- data/lib/generators/dispatch_policy/install/templates/dispatch_tick_loop_job.rb.tt +53 -0
- data/lib/generators/dispatch_policy/install/templates/initializer.rb.tt +11 -0
- metadata +101 -43
- data/CHANGELOG.md +0 -43
- data/app/models/dispatch_policy/partition_inflight_count.rb +0 -42
- data/app/models/dispatch_policy/partition_observation.rb +0 -76
- data/app/models/dispatch_policy/throttle_bucket.rb +0 -41
- data/db/migrate/20260424000001_create_dispatch_policy_tables.rb +0 -80
- data/db/migrate/20260424000002_create_adaptive_concurrency_stats.rb +0 -22
- data/db/migrate/20260424000003_create_adaptive_concurrency_samples.rb +0 -25
- data/db/migrate/20260424000004_rename_samples_to_partition_observations.rb +0 -32
- data/db/migrate/20260425000001_add_duration_to_partition_observations.rb +0 -8
- data/lib/dispatch_policy/active_job_perform_all_later_patch.rb +0 -32
- data/lib/dispatch_policy/dispatch_context.rb +0 -53
- data/lib/dispatch_policy/dispatchable.rb +0 -123
- data/lib/dispatch_policy/gates/fair_interleave.rb +0 -32
- data/lib/dispatch_policy/gates/global_cap.rb +0 -26
|
@@ -1,249 +1,165 @@
|
|
|
1
|
-
<
|
|
2
|
-
<h1><%= @policy_name %> <small class="muted"><%= @job_class.name %></small></h1>
|
|
1
|
+
<h1>Policy <code><%= @policy_name %></code></h1>
|
|
3
2
|
|
|
4
|
-
<
|
|
5
|
-
<div class="
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
</div>
|
|
10
|
-
<div class="summary-item">
|
|
11
|
-
<div class="label">Admitted</div>
|
|
12
|
-
<div class="value"><%= @admitted_count %></div>
|
|
13
|
-
</div>
|
|
14
|
-
<div class="summary-item">
|
|
15
|
-
<div class="label">Completed (24h)</div>
|
|
16
|
-
<div class="value"><%= @completed_24h %></div>
|
|
17
|
-
</div>
|
|
18
|
-
<div class="summary-item">
|
|
19
|
-
<div class="label">Gates</div>
|
|
20
|
-
<div class="value" style="font-size: 1em;">
|
|
21
|
-
<% (@policy&.gates || []).each do |g| %>
|
|
22
|
-
<span class="badge"><%= g.name %></span>
|
|
23
|
-
<% end %>
|
|
24
|
-
</div>
|
|
25
|
-
</div>
|
|
26
|
-
</div>
|
|
27
|
-
|
|
28
|
-
<% if @adaptive_global.any? { |v| !v.nil? } %>
|
|
29
|
-
<h2>Adaptive — EWMA queue lag (avg across partitions, last hour)</h2>
|
|
30
|
-
<div class="chart-global">
|
|
31
|
-
<canvas data-chart="line"
|
|
32
|
-
data-label="avg ewma_latency_ms"
|
|
33
|
-
data-labels='<%= @chart_labels.to_json %>'
|
|
34
|
-
data-values='<%= @adaptive_global.to_json %>'
|
|
35
|
-
data-counts='<%= @completions_global.to_json %>'></canvas>
|
|
36
|
-
</div>
|
|
37
|
-
<% end %>
|
|
3
|
+
<section class="dp-stats">
|
|
4
|
+
<div class="dp-stat"><span class="dp-stat-label">Partitions</span><span class="dp-stat-value"><%= format_count(@totals[:partitions]) %></span></div>
|
|
5
|
+
<div class="dp-stat"><span class="dp-stat-label">Pending</span><span class="dp-stat-value"><%= format_count(@totals[:pending]) %></span></div>
|
|
6
|
+
<div class="dp-stat"><span class="dp-stat-label">In flight</span><span class="dp-stat-value"><%= format_count(@totals[:in_flight]) %></span></div>
|
|
7
|
+
</section>
|
|
38
8
|
|
|
39
|
-
|
|
9
|
+
<%= render "dispatch_policy/shared/hints", hints: @hints %>
|
|
10
|
+
<%= render "dispatch_policy/shared/capacity",
|
|
11
|
+
capacity: @capacity,
|
|
12
|
+
pending_trend: @pending_trend,
|
|
13
|
+
pending_buckets: @sparkline %>
|
|
40
14
|
|
|
41
|
-
|
|
42
|
-
<
|
|
43
|
-
|
|
44
|
-
<table data-watched-table data-policy-name="<%= @policy_name %>">
|
|
15
|
+
<section class="dp-section">
|
|
16
|
+
<h2>Throughput</h2>
|
|
17
|
+
<table class="dp-table">
|
|
45
18
|
<thead>
|
|
46
|
-
<tr>
|
|
47
|
-
<th></th>
|
|
48
|
-
<th>Source</th>
|
|
49
|
-
<th>Partition</th>
|
|
50
|
-
<th class="text-end">Pending (eligible)</th>
|
|
51
|
-
<th class="text-end">Pending (scheduled)</th>
|
|
52
|
-
<th class="text-end">In-flight</th>
|
|
53
|
-
<th class="text-end">Completed (24h)</th>
|
|
54
|
-
<th class="text-end">Adaptive cap</th>
|
|
55
|
-
<th class="text-end">EWMA latency (ms)</th>
|
|
56
|
-
</tr>
|
|
19
|
+
<tr><th>Window</th><th class="dp-num">Admitted</th><th class="dp-num">Ticks</th><th class="dp-num">Avg tick</th><th class="dp-num">Max tick</th><th class="dp-num">Forward fail</th><th class="dp-num">Partitions seen</th></tr>
|
|
57
20
|
</thead>
|
|
58
21
|
<tbody>
|
|
59
|
-
<% @
|
|
60
|
-
|
|
61
|
-
<td><button type="button" class="chip" data-unwatch="<%= row[:partition] %>" title="Stop watching">×</button></td>
|
|
62
|
-
<td><%= row[:source] %></td>
|
|
63
|
-
<td><code><%= row[:partition] %></code></td>
|
|
64
|
-
<td class="text-end"><%= row[:eligible] %></td>
|
|
65
|
-
<td class="text-end"><%= row[:scheduled] %></td>
|
|
66
|
-
<td class="text-end"><%= row[:in_flight] %></td>
|
|
67
|
-
<td class="text-end"><%= row[:completed_24h] %></td>
|
|
68
|
-
<td class="text-end">
|
|
69
|
-
<% if row[:current_max] %><strong><%= row[:current_max] %></strong><% else %><span class="muted">—</span><% end %>
|
|
70
|
-
</td>
|
|
71
|
-
<td class="text-end">
|
|
72
|
-
<% if row[:ewma_latency_ms] %><%= row[:ewma_latency_ms] %><% else %><span class="muted">—</span><% end %>
|
|
73
|
-
</td>
|
|
74
|
-
</tr>
|
|
75
|
-
<%
|
|
76
|
-
series = @adaptive_samples[row[:partition]] || Array.new(@chart_slots.size)
|
|
77
|
-
counts = @completions_samples[row[:partition]] || Array.new(@chart_slots.size, 0)
|
|
78
|
-
has_samples = series.any? { |v| !v.nil? } || counts.any? { |c| c.positive? }
|
|
79
|
-
%>
|
|
22
|
+
<% @windows.each do |label, m| %>
|
|
23
|
+
<% fail_pct = m[:jobs_admitted].positive? ? (m[:forward_failures].to_f / m[:jobs_admitted] * 100) : 0.0 %>
|
|
80
24
|
<tr>
|
|
81
|
-
<td
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
<div class="sparkline-wrap sparkline-empty"><span class="muted">no completions in the last hour yet</span></div>
|
|
25
|
+
<td><strong><%= label %></strong></td>
|
|
26
|
+
<td class="dp-num"><%= format_count(m[:jobs_admitted]) %></td>
|
|
27
|
+
<td class="dp-num"><%= format_count(m[:ticks]) %></td>
|
|
28
|
+
<td class="dp-num"><%= format_duration_ms(m[:avg_duration_ms]) %></td>
|
|
29
|
+
<td class="dp-num"><%= format_duration_ms(m[:max_duration_ms]) %></td>
|
|
30
|
+
<td class="dp-num <%= "dp-warn" if fail_pct >= 1.0 %>">
|
|
31
|
+
<%= format_count(m[:forward_failures]) %>
|
|
32
|
+
<% if m[:jobs_admitted].positive? && m[:forward_failures].positive? %>
|
|
33
|
+
<span style="color:#9ca3af; font-size:11px;">(<%= format("%.2f%%", fail_pct) %>)</span>
|
|
91
34
|
<% end %>
|
|
92
35
|
</td>
|
|
36
|
+
<td class="dp-num"><%= format_count(m[:partitions_seen]) %></td>
|
|
93
37
|
</tr>
|
|
94
38
|
<% end %>
|
|
95
39
|
</tbody>
|
|
96
40
|
</table>
|
|
97
|
-
<% end %>
|
|
98
41
|
|
|
99
|
-
|
|
42
|
+
<% spark_admit = @sparkline.map { |b| b[:jobs_admitted] } %>
|
|
43
|
+
<% if spark_admit.any? %>
|
|
44
|
+
<p class="dp-spark">
|
|
45
|
+
Last 30 minutes (1-minute buckets): <code><%= sparkline(spark_admit, width: 30) %></code>
|
|
46
|
+
peak: <%= format_count(spark_admit.max) %> jobs/min
|
|
47
|
+
</p>
|
|
48
|
+
<% end %>
|
|
49
|
+
</section>
|
|
100
50
|
|
|
101
|
-
|
|
102
|
-
<
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
51
|
+
<section class="dp-section">
|
|
52
|
+
<h2>Round-trip</h2>
|
|
53
|
+
<% if @round_trip[:active_partitions].zero? %>
|
|
54
|
+
<p class="dp-empty">No active partitions with pending jobs in this policy.</p>
|
|
55
|
+
<% else %>
|
|
56
|
+
<div class="dp-stats">
|
|
57
|
+
<div class="dp-stat"><span class="dp-stat-label">Active w/ pending</span><span class="dp-stat-value"><%= format_count(@round_trip[:active_partitions]) %></span></div>
|
|
58
|
+
<div class="dp-stat"><span class="dp-stat-label">Never checked</span><span class="dp-stat-value <%= "dp-warn" if @round_trip[:never_checked].positive? %>"><%= format_count(@round_trip[:never_checked]) %></span></div>
|
|
59
|
+
<div class="dp-stat"><span class="dp-stat-label">In backoff</span><span class="dp-stat-value"><%= format_count(@round_trip[:in_backoff]) %></span></div>
|
|
60
|
+
<div class="dp-stat"><span class="dp-stat-label">Oldest age</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:oldest_age_seconds]) %></span></div>
|
|
61
|
+
<div class="dp-stat"><span class="dp-stat-label">P50</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:p50_age_seconds]) %></span></div>
|
|
62
|
+
<div class="dp-stat"><span class="dp-stat-label">P95</span><span class="dp-stat-value"><%= format_duration_seconds(@round_trip[:p95_age_seconds]) %></span></div>
|
|
63
|
+
</div>
|
|
64
|
+
<% end %>
|
|
65
|
+
</section>
|
|
108
66
|
|
|
109
|
-
<
|
|
110
|
-
|
|
111
|
-
|
|
67
|
+
<section class="dp-section">
|
|
68
|
+
<h2>Denials (last 15m)</h2>
|
|
69
|
+
<% if @denied_reasons.empty? %>
|
|
70
|
+
<p class="dp-empty">No denials recorded in the window.</p>
|
|
112
71
|
<% else %>
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
72
|
+
<table class="dp-table">
|
|
73
|
+
<thead><tr><th>Reason</th><th class="dp-num">Count</th></tr></thead>
|
|
74
|
+
<tbody>
|
|
75
|
+
<% @denied_reasons.each do |reason, count| %>
|
|
76
|
+
<tr><td><code><%= reason %></code></td><td class="dp-num"><%= format_count(count) %></td></tr>
|
|
77
|
+
<% end %>
|
|
78
|
+
</tbody>
|
|
79
|
+
</table>
|
|
80
|
+
<p class="dp-hint">
|
|
81
|
+
<code>throttle_empty</code> dominating? Raise the policy's <code>rate</code> or accept the throttle and shard tenants.
|
|
82
|
+
<code>concurrency_full</code> dominating? Raise <code>max</code> on the concurrency gate or scale the worker pool.
|
|
83
|
+
</p>
|
|
84
|
+
<% end %>
|
|
85
|
+
</section>
|
|
86
|
+
|
|
87
|
+
<section class="dp-section">
|
|
88
|
+
<h2>Gates</h2>
|
|
89
|
+
<% if @policy_object %>
|
|
90
|
+
<ul class="dp-list">
|
|
91
|
+
<% @policy_object.gates.each do |gate| %>
|
|
92
|
+
<li><strong><%= gate.name %></strong> (<%= gate.class.name.demodulize %>)</li>
|
|
121
93
|
<% end %>
|
|
122
|
-
|
|
94
|
+
</ul>
|
|
95
|
+
<p>Retry strategy: <code><%= @policy_object.retry_strategy %></code></p>
|
|
96
|
+
<% else %>
|
|
97
|
+
<p class="dp-empty">Policy <code><%= @policy_name %></code> is not registered in this process.</p>
|
|
98
|
+
<% end %>
|
|
99
|
+
</section>
|
|
123
100
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
arrow = active ? (@partition_dir == "asc" ? " ▲" : " ▼") : ""
|
|
132
|
-
content_tag(:th, class: (align_end ? "text-end" : nil)) do
|
|
133
|
-
link_to "#{label}#{arrow}",
|
|
134
|
-
url_for(sort: col, dir: next_dir, q: @partition_search.presence, page: nil, watch: params[:watch]),
|
|
135
|
-
class: (active ? "sorted" : nil)
|
|
136
|
-
end
|
|
137
|
-
}
|
|
138
|
-
%>
|
|
139
|
-
<table data-policy-name="<%= @policy_name %>">
|
|
140
|
-
<thead>
|
|
101
|
+
<section class="dp-section">
|
|
102
|
+
<h2>Top partitions by lifetime admitted</h2>
|
|
103
|
+
<% if @top_admitted.any? %>
|
|
104
|
+
<table class="dp-table">
|
|
105
|
+
<thead><tr><th>Partition key</th><th class="dp-num">Lifetime admitted</th><th class="dp-num">Pending</th><th>Last admit</th></tr></thead>
|
|
106
|
+
<tbody>
|
|
107
|
+
<% @top_admitted.each do |p| %>
|
|
141
108
|
<tr>
|
|
142
|
-
<
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
<%= sort_link.call("In-flight", "in_flight", align_end: true) %>
|
|
147
|
-
<%= sort_link.call("Completed (24h)", "completed_24h", align_end: true) %>
|
|
148
|
-
<%= sort_link.call("Last enqueue", "last_enqueued_at") %>
|
|
149
|
-
<%= sort_link.call("Last dispatch", "last_dispatched_at") %>
|
|
109
|
+
<td><%= link_to p.partition_key, partition_path(p), class: "dp-link" %></td>
|
|
110
|
+
<td class="dp-num"><%= format_count(p.total_admitted) %></td>
|
|
111
|
+
<td class="dp-num"><%= format_count(p.pending_count) %></td>
|
|
112
|
+
<td><%= format_time(p.last_admit_at) %></td>
|
|
150
113
|
</tr>
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
<td>
|
|
157
|
-
<% if @watched_keys.include?(row[:partition]) %>
|
|
158
|
-
<span class="chip on" title="Watched">✓</span>
|
|
159
|
-
<% else %>
|
|
160
|
-
<button type="button" class="chip" data-watch="<%= row[:partition] %>" data-turbo-frame="_top" title="Watch this partition">+</button>
|
|
161
|
-
<% end %>
|
|
162
|
-
</td>
|
|
163
|
-
<td><%= row[:source] %></td>
|
|
164
|
-
<td><code><%= row[:partition] %></code></td>
|
|
165
|
-
<td class="text-end"><%= pending_total %></td>
|
|
166
|
-
<td class="text-end"><%= row[:in_flight] %></td>
|
|
167
|
-
<td class="text-end"><%= row[:completed_24h] %></td>
|
|
168
|
-
<td>
|
|
169
|
-
<% if row[:last_enqueued_at] %>
|
|
170
|
-
<span title="<%= row[:last_enqueued_at] %>"><%= time_ago_in_words(row[:last_enqueued_at]) %> ago</span>
|
|
171
|
-
<% else %>
|
|
172
|
-
<span class="muted">—</span>
|
|
173
|
-
<% end %>
|
|
174
|
-
</td>
|
|
175
|
-
<td>
|
|
176
|
-
<% if row[:last_dispatched_at] %>
|
|
177
|
-
<span title="<%= row[:last_dispatched_at] %>"><%= time_ago_in_words(row[:last_dispatched_at]) %> ago</span>
|
|
178
|
-
<% else %>
|
|
179
|
-
<span class="muted">—</span>
|
|
180
|
-
<% end %>
|
|
181
|
-
</td>
|
|
182
|
-
</tr>
|
|
183
|
-
<% end %>
|
|
184
|
-
</tbody>
|
|
185
|
-
</table>
|
|
186
|
-
|
|
187
|
-
<% total_pages = (@partition_total_list / DispatchPolicy::PoliciesController::PARTITION_LIST_PAGE_SIZE.to_f).ceil %>
|
|
188
|
-
<% if total_pages > 1 %>
|
|
189
|
-
<% page_params = { q: @partition_search.presence, watch: params[:watch], sort: params[:sort], dir: params[:dir] } %>
|
|
190
|
-
<div class="pagination">
|
|
191
|
-
<% if @partition_page > 1 %>
|
|
192
|
-
<%= link_to "← prev", url_for(page_params.merge(page: @partition_page - 1)) %>
|
|
193
|
-
<% end %>
|
|
194
|
-
<span class="muted">page <%= @partition_page %> / <%= total_pages %></span>
|
|
195
|
-
<% if @partition_page < total_pages %>
|
|
196
|
-
<%= link_to "next →", url_for(page_params.merge(page: @partition_page + 1)) %>
|
|
197
|
-
<% end %>
|
|
198
|
-
</div>
|
|
199
|
-
<% end %>
|
|
200
|
-
<% end %>
|
|
114
|
+
<% end %>
|
|
115
|
+
</tbody>
|
|
116
|
+
</table>
|
|
117
|
+
<% else %>
|
|
118
|
+
<p class="dp-empty">No partitions have admitted any jobs yet.</p>
|
|
201
119
|
<% end %>
|
|
202
|
-
</
|
|
120
|
+
</section>
|
|
203
121
|
|
|
204
|
-
<
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
<thead>
|
|
210
|
-
<tr><th>Gate</th><th>Partition</th><th>Tokens</th><th>Refilled at</th></tr>
|
|
211
|
-
</thead>
|
|
212
|
-
<tbody>
|
|
213
|
-
<% @throttle_buckets.each do |b| %>
|
|
122
|
+
<section class="dp-section">
|
|
123
|
+
<h2>Top partitions by pending</h2>
|
|
124
|
+
<% if @partitions.any? %>
|
|
125
|
+
<table class="dp-table">
|
|
126
|
+
<thead>
|
|
214
127
|
<tr>
|
|
215
|
-
<
|
|
216
|
-
<
|
|
217
|
-
<
|
|
218
|
-
<td><%= b.refilled_at.to_s(:short) rescue b.refilled_at %></td>
|
|
128
|
+
<th>Partition key</th><th>Status</th>
|
|
129
|
+
<th class="dp-num">Pending</th>
|
|
130
|
+
<th>Next eligible</th><th>Last admit</th><th></th>
|
|
219
131
|
</tr>
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
132
|
+
</thead>
|
|
133
|
+
<tbody>
|
|
134
|
+
<% @partitions.each do |p| %>
|
|
135
|
+
<tr>
|
|
136
|
+
<td><%= link_to p.partition_key, partition_path(p), class: "dp-link" %></td>
|
|
137
|
+
<td><%= p.status %></td>
|
|
138
|
+
<td class="dp-num"><%= format_count(p.pending_count) %></td>
|
|
139
|
+
<td><%= format_time(p.next_eligible_at) %></td>
|
|
140
|
+
<td><%= format_time(p.last_admit_at) %></td>
|
|
141
|
+
<td><%= link_to "→", partition_path(p), class: "dp-link" %></td>
|
|
142
|
+
</tr>
|
|
143
|
+
<% end %>
|
|
144
|
+
</tbody>
|
|
145
|
+
</table>
|
|
146
|
+
<% else %>
|
|
147
|
+
<p class="dp-empty">No partitions for this policy yet.</p>
|
|
148
|
+
<% end %>
|
|
149
|
+
</section>
|
|
224
150
|
|
|
225
|
-
<
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
</
|
|
236
|
-
<
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
<td><%= job.dedupe_key ? content_tag(:code, job.dedupe_key) : content_tag(:span, '—', class: 'muted') %></td>
|
|
241
|
-
<td><%= job.round_robin_key ? content_tag(:code, job.round_robin_key) : content_tag(:span, '—', class: 'muted') %></td>
|
|
242
|
-
<td><%= job.priority %></td>
|
|
243
|
-
<td><%= job.staged_at %></td>
|
|
244
|
-
<td><%= job.not_before_at || content_tag(:span, '—', class: 'muted') %></td>
|
|
245
|
-
</tr>
|
|
246
|
-
<% end %>
|
|
247
|
-
</tbody>
|
|
248
|
-
</table>
|
|
249
|
-
<% end %>
|
|
151
|
+
<section class="dp-section">
|
|
152
|
+
<h2>Actions</h2>
|
|
153
|
+
<%= button_to "Pause all partitions", pause_policy_path(@policy_name), class: "dp-btn", method: :post, form: { class: "dp-form-inline" } %>
|
|
154
|
+
<%= button_to "Resume all partitions", resume_policy_path(@policy_name), class: "dp-btn dp-btn-ok", method: :post, form: { class: "dp-form-inline" } %>
|
|
155
|
+
<%= button_to "Drain policy", drain_policy_path(@policy_name),
|
|
156
|
+
class: "dp-btn dp-btn-warn",
|
|
157
|
+
method: :post,
|
|
158
|
+
form: { class: "dp-form-inline",
|
|
159
|
+
onsubmit: "return confirm('Force-admit every staged job across every partition of this policy, bypassing all gates?');" } %>
|
|
160
|
+
<p class="dp-hint">
|
|
161
|
+
<strong>Pause</strong> stops admission but keeps staging — the queue keeps filling, in-flight jobs finish.
|
|
162
|
+
<strong>Drain</strong> empties the staging table by force-admitting every job (bypassing gates).
|
|
163
|
+
Capped at 10,000 jobs per click — click again for more.
|
|
164
|
+
</p>
|
|
165
|
+
</section>
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
<%
|
|
2
|
+
capacity = local_assigns.fetch(:capacity)
|
|
3
|
+
pending_trend = local_assigns.fetch(:pending_trend, :flat)
|
|
4
|
+
pending_buckets = local_assigns[:pending_buckets]
|
|
5
|
+
|
|
6
|
+
jps = capacity[:admitted_per_second] || (capacity[:admitted_per_minute].to_f / 60.0)
|
|
7
|
+
target = capacity[:adapter_target_jps]
|
|
8
|
+
ratio_jps = (target && target.positive?) ? jps / target : nil
|
|
9
|
+
tick_ratio = (capacity[:tick_max_duration_ms].to_i.positive? ?
|
|
10
|
+
capacity[:avg_tick_ms].to_f / capacity[:tick_max_duration_ms] : nil)
|
|
11
|
+
|
|
12
|
+
arrow = case pending_trend
|
|
13
|
+
when :up then "↑"
|
|
14
|
+
when :down then "↓"
|
|
15
|
+
else "→"
|
|
16
|
+
end
|
|
17
|
+
%>
|
|
18
|
+
|
|
19
|
+
<section class="dp-section">
|
|
20
|
+
<h2>Capacity headroom</h2>
|
|
21
|
+
<div class="dp-stats">
|
|
22
|
+
<div class="dp-stat">
|
|
23
|
+
<span class="dp-stat-label">Admit rate (1m)</span>
|
|
24
|
+
<span class="dp-stat-value"><%= format("%.1f", jps) %> jobs/sec</span>
|
|
25
|
+
</div>
|
|
26
|
+
<% if ratio_jps %>
|
|
27
|
+
<div class="dp-stat">
|
|
28
|
+
<span class="dp-stat-label">Of adapter ceiling (<%= target %>/sec)</span>
|
|
29
|
+
<span class="dp-stat-value <%= "dp-warn" if ratio_jps >= 0.7 %>">
|
|
30
|
+
<%= format("%.0f%%", ratio_jps * 100) %>
|
|
31
|
+
</span>
|
|
32
|
+
</div>
|
|
33
|
+
<% end %>
|
|
34
|
+
<div class="dp-stat">
|
|
35
|
+
<span class="dp-stat-label">Avg tick</span>
|
|
36
|
+
<span class="dp-stat-value"><%= format_duration_ms(capacity[:avg_tick_ms]) %></span>
|
|
37
|
+
</div>
|
|
38
|
+
<% if tick_ratio %>
|
|
39
|
+
<div class="dp-stat">
|
|
40
|
+
<span class="dp-stat-label">Of tick_max_duration</span>
|
|
41
|
+
<span class="dp-stat-value <%= "dp-warn" if tick_ratio >= 0.6 %>">
|
|
42
|
+
<%= format("%.0f%%", tick_ratio * 100) %>
|
|
43
|
+
</span>
|
|
44
|
+
</div>
|
|
45
|
+
<% end %>
|
|
46
|
+
<div class="dp-stat">
|
|
47
|
+
<span class="dp-stat-label">Pending trend (30m)</span>
|
|
48
|
+
<span class="dp-stat-value <%= "dp-warn" if pending_trend == :up %>">
|
|
49
|
+
<%= arrow %> <%= pending_trend %>
|
|
50
|
+
</span>
|
|
51
|
+
</div>
|
|
52
|
+
</div>
|
|
53
|
+
|
|
54
|
+
<% if pending_buckets && pending_buckets.any? %>
|
|
55
|
+
<p class="dp-spark">
|
|
56
|
+
Pending across the last 30 min:
|
|
57
|
+
<code><%= sparkline(pending_buckets.map { |b| b[:pending_total] }, width: 30) %></code>
|
|
58
|
+
</p>
|
|
59
|
+
<% end %>
|
|
60
|
+
<% if target.nil? %>
|
|
61
|
+
<p class="dp-hint">
|
|
62
|
+
Set <code>DispatchPolicy.config.adapter_throughput_target = 3500</code>
|
|
63
|
+
(jobs/sec) in your initializer to see the admit rate as a percentage.
|
|
64
|
+
Locally we measure ~3500/sec per worker against good_job; calibrate to your prod adapter.
|
|
65
|
+
</p>
|
|
66
|
+
<% end %>
|
|
67
|
+
</section>
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
<% if hints.any? %>
|
|
2
|
+
<section class="dp-section">
|
|
3
|
+
<h2>Hints</h2>
|
|
4
|
+
<ul class="dp-list dp-hint-list">
|
|
5
|
+
<% hints.each do |hint| %>
|
|
6
|
+
<li class="dp-hint-<%= hint.level %>">
|
|
7
|
+
<span class="dp-hint-badge"><%= hint.level.to_s.upcase %></span>
|
|
8
|
+
<%= hint.message %>
|
|
9
|
+
</li>
|
|
10
|
+
<% end %>
|
|
11
|
+
</ul>
|
|
12
|
+
</section>
|
|
13
|
+
<% end %>
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
<tr id="partition_<%= partition.id %>" class="<%= "dp-row-paused" if partition.paused? %>">
|
|
2
|
+
<td><%= link_to partition.policy_name, policy_path(partition.policy_name), class: "dp-link" %></td>
|
|
3
|
+
<td><code><%= partition.shard %></code></td>
|
|
4
|
+
<td><%= partition.queue_name || "—" %></td>
|
|
5
|
+
<td><%= link_to partition.partition_key, partition_path(partition), class: "dp-link" %></td>
|
|
6
|
+
<td><%= partition.status %></td>
|
|
7
|
+
<td class="dp-num"><%= format_count(partition.pending_count) %></td>
|
|
8
|
+
<td class="dp-num"><%= format_count(partition.total_admitted) %></td>
|
|
9
|
+
<td><%= format_time(partition.next_eligible_at) %></td>
|
|
10
|
+
<td><%= format_time(partition.last_admit_at) %></td>
|
|
11
|
+
<td><%= format_time(partition.last_enqueued_at) %></td>
|
|
12
|
+
</tr>
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
<h1>Staged job #<%= @job.id %></h1>
|
|
2
|
+
|
|
3
|
+
<section class="dp-stats">
|
|
4
|
+
<div class="dp-stat"><span class="dp-stat-label">Policy</span><span class="dp-stat-value"><%= @job.policy_name %></span></div>
|
|
5
|
+
<div class="dp-stat"><span class="dp-stat-label">Class</span><span class="dp-stat-value"><code><%= @job.job_class %></code></span></div>
|
|
6
|
+
<div class="dp-stat"><span class="dp-stat-label">Queue</span><span class="dp-stat-value"><%= @job.queue_name || "—" %></span></div>
|
|
7
|
+
<div class="dp-stat"><span class="dp-stat-label">Priority</span><span class="dp-stat-value"><%= @job.priority %></span></div>
|
|
8
|
+
</section>
|
|
9
|
+
|
|
10
|
+
<section class="dp-section">
|
|
11
|
+
<h2>Partition</h2>
|
|
12
|
+
<p><code><%= @job.partition_key %></code></p>
|
|
13
|
+
</section>
|
|
14
|
+
|
|
15
|
+
<section class="dp-section">
|
|
16
|
+
<h2>Timing</h2>
|
|
17
|
+
<ul class="dp-list">
|
|
18
|
+
<li>Enqueued: <%= format_time(@job.enqueued_at) %></li>
|
|
19
|
+
<li>Scheduled: <%= format_time(@job.scheduled_at) %></li>
|
|
20
|
+
</ul>
|
|
21
|
+
</section>
|
|
22
|
+
|
|
23
|
+
<section class="dp-section">
|
|
24
|
+
<h2>Context snapshot</h2>
|
|
25
|
+
<pre class="dp-json"><%= JSON.pretty_generate(@job.context || {}) %></pre>
|
|
26
|
+
</section>
|
|
27
|
+
|
|
28
|
+
<section class="dp-section">
|
|
29
|
+
<h2>ActiveJob payload</h2>
|
|
30
|
+
<pre class="dp-json"><%= JSON.pretty_generate(@job.job_data || {}) %></pre>
|
|
31
|
+
</section>
|