patient_http-sidekiq 1.3.0 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +22 -0
- data/README.md +105 -2
- data/VERSION +1 -1
- data/lib/patient_http/sidekiq/configuration.rb +231 -1
- data/lib/patient_http/sidekiq/direct_task_handler.rb +3 -1
- data/lib/patient_http/sidekiq/processor_observer.rb +151 -24
- data/lib/patient_http/sidekiq/redis_pool.rb +88 -0
- data/lib/patient_http/sidekiq/request_executor.rb +33 -5
- data/lib/patient_http/sidekiq/request_worker.rb +5 -2
- data/lib/patient_http/sidekiq/stats.rb +230 -33
- data/lib/patient_http/sidekiq/task_handler.rb +9 -3
- data/lib/patient_http/sidekiq/task_monitor.rb +410 -121
- data/lib/patient_http/sidekiq/task_monitor_thread.rb +36 -3
- data/lib/patient_http/sidekiq/web_ui/assets/patient-http/css/patient_http.css +29 -71
- data/lib/patient_http/sidekiq/web_ui/locales/ar.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/cs.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/da.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/de.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/el.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/en.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/es.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/fa.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/fr.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/gd.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/he.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/hi.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/it.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/ja.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/ko.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/lt.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/nb.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/nl.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/pl.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/pt-BR.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/pt.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/ru.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/sv.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/ta.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/tr.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/uk.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/ur.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/vi.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/zh-CN.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/locales/zh-TW.yml +10 -5
- data/lib/patient_http/sidekiq/web_ui/views/patient_http.html.erb +99 -44
- data/lib/patient_http/sidekiq/web_ui.rb +53 -1
- data/lib/patient_http/sidekiq.rb +236 -31
- data/patient_http-sidekiq.gemspec +1 -1
- metadata +5 -4
|
@@ -31,60 +31,115 @@
|
|
|
31
31
|
<span><%= avg_duration %></span>
|
|
32
32
|
</div>
|
|
33
33
|
</div>
|
|
34
|
+
<div class="stat-card">
|
|
35
|
+
<div class="stat-label"><%= t('patient_http_max_capacity') %></div>
|
|
36
|
+
<div class="stat-value"><%= number_with_delimiter(max_capacity) %></div>
|
|
37
|
+
</div>
|
|
38
|
+
<div class="stat-card">
|
|
39
|
+
<div class="stat-label"><%= t('patient_http_currently_inflight') %></div>
|
|
40
|
+
<div class="stat-value"><%= number_with_delimiter(current_inflight) %></div>
|
|
41
|
+
</div>
|
|
42
|
+
<div class="stat-card">
|
|
43
|
+
<div class="stat-label"><%= t('patient_http_capacity_utilization') %></div>
|
|
44
|
+
<div class="stat-value"><%= utilization %>%</div>
|
|
45
|
+
</div>
|
|
34
46
|
</div>
|
|
35
47
|
|
|
36
|
-
<!--
|
|
37
|
-
|
|
38
|
-
<
|
|
39
|
-
|
|
40
|
-
<div class="
|
|
41
|
-
<
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
48
|
+
<!-- Processor Breakdown -->
|
|
49
|
+
<% unless processors.empty? %>
|
|
50
|
+
<div class="breakdown-section">
|
|
51
|
+
<h3><%= t('patient_http_processors') %></h3>
|
|
52
|
+
<div class="table-scroll">
|
|
53
|
+
<table class="table">
|
|
54
|
+
<thead>
|
|
55
|
+
<tr>
|
|
56
|
+
<th><%= t('patient_http_processor') %></th>
|
|
57
|
+
<th><%= t('patient_http_inflight') %></th>
|
|
58
|
+
<th><%= t('patient_http_peak_inflight') %></th>
|
|
59
|
+
<th><%= t('patient_http_max_capacity') %></th>
|
|
60
|
+
<th><%= t('patient_http_utilization') %></th>
|
|
61
|
+
<th><%= t('patient_http_total_requests') %></th>
|
|
62
|
+
<th><%= t('patient_http_errors') %></th>
|
|
63
|
+
<th><%= t('patient_http_max_capacity_exceeded') %></th>
|
|
64
|
+
<th><%= t('patient_http_avg_duration') %></th>
|
|
65
|
+
</tr>
|
|
66
|
+
</thead>
|
|
67
|
+
<tbody>
|
|
68
|
+
<% processors.each do |processor| %>
|
|
69
|
+
<tr>
|
|
70
|
+
<td><%= h(processor[:name]) %></td>
|
|
71
|
+
<td><%= number_with_delimiter(processor[:inflight]) %></td>
|
|
72
|
+
<td><%= number_with_delimiter(processor[:peak_inflight]) %></td>
|
|
73
|
+
<td><%= number_with_delimiter(processor[:max_capacity]) %></td>
|
|
74
|
+
<td><%= processor[:utilization] %>%</td>
|
|
75
|
+
<td><%= number_with_delimiter(processor[:requests]) %></td>
|
|
76
|
+
<td><%= number_with_delimiter(processor[:errors]) %></td>
|
|
77
|
+
<td><%= number_with_delimiter(processor[:max_capacity_exceeded]) %></td>
|
|
78
|
+
<td><%= processor[:avg_duration] %></td>
|
|
79
|
+
</tr>
|
|
80
|
+
<% end %>
|
|
81
|
+
</tbody>
|
|
82
|
+
</table>
|
|
54
83
|
</div>
|
|
55
84
|
</div>
|
|
85
|
+
<% end %>
|
|
56
86
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
<
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
<
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
<% if processes.empty? %>
|
|
71
|
-
<tr class="empty-state">
|
|
72
|
-
<td colspan="4"><%= t('patient_http_no_active_processes') %></td>
|
|
87
|
+
<!-- In-Flight Requests -->
|
|
88
|
+
<% unless inflight.empty? %>
|
|
89
|
+
<div class="breakdown-section">
|
|
90
|
+
<h3><%= t('patient_http_inflight_requests') %></h3>
|
|
91
|
+
<div class="table-scroll">
|
|
92
|
+
<table class="table">
|
|
93
|
+
<thead>
|
|
94
|
+
<tr>
|
|
95
|
+
<th><%= t('patient_http_age') %></th>
|
|
96
|
+
<th><%= t('patient_http_method') %></th>
|
|
97
|
+
<th><%= t('patient_http_url') %></th>
|
|
98
|
+
<th><%= t('patient_http_processor') %></th>
|
|
99
|
+
<th><%= t('patient_http_process') %></th>
|
|
73
100
|
</tr>
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
101
|
+
</thead>
|
|
102
|
+
<tbody>
|
|
103
|
+
<% inflight.each do |request| %>
|
|
77
104
|
<tr>
|
|
78
|
-
<td><%=
|
|
79
|
-
<td><%=
|
|
80
|
-
<td><%=
|
|
81
|
-
<td><%=
|
|
105
|
+
<td><%= request[:age] %></td>
|
|
106
|
+
<td><%= h(request[:http_method].to_s.upcase) %></td>
|
|
107
|
+
<td class="url"><%= h(request[:url]) %></td>
|
|
108
|
+
<td><%= h(request[:processor]) %></td>
|
|
109
|
+
<td><%= h(request[:process_id]) %></td>
|
|
82
110
|
</tr>
|
|
83
111
|
<% end %>
|
|
84
|
-
|
|
85
|
-
</
|
|
86
|
-
</
|
|
112
|
+
</tbody>
|
|
113
|
+
</table>
|
|
114
|
+
</div>
|
|
87
115
|
</div>
|
|
116
|
+
<% end %>
|
|
117
|
+
|
|
118
|
+
<!-- Sidekiq Process Breakdown -->
|
|
119
|
+
<div class="breakdown-section">
|
|
120
|
+
<h3><%= t('patient_http_sidekiq_processes') %></h3>
|
|
121
|
+
<table class="table">
|
|
122
|
+
<thead>
|
|
123
|
+
<tr>
|
|
124
|
+
<th><%= t('patient_http_process') %></th>
|
|
125
|
+
<th><%= t('patient_http_inflight') %></th>
|
|
126
|
+
</tr>
|
|
127
|
+
</thead>
|
|
128
|
+
<tbody>
|
|
129
|
+
<% if processes.empty? %>
|
|
130
|
+
<tr class="empty-state">
|
|
131
|
+
<td colspan="2"><%= t('patient_http_no_active_processes') %></td>
|
|
132
|
+
</tr>
|
|
133
|
+
<% else %>
|
|
134
|
+
<% processes.each do |process_id, info| %>
|
|
135
|
+
<tr>
|
|
136
|
+
<td><%= h(process_id) %></td>
|
|
137
|
+
<td><%= number_with_delimiter(info[:inflight]) %></td>
|
|
138
|
+
</tr>
|
|
139
|
+
<% end %>
|
|
140
|
+
<% end %>
|
|
141
|
+
</tbody>
|
|
142
|
+
</table>
|
|
88
143
|
</div>
|
|
89
144
|
|
|
90
145
|
<!-- HTTP Status Codes Breakdown -->
|
|
@@ -20,6 +20,10 @@ module PatientHttp
|
|
|
20
20
|
ROOT = File.join(__dir__, "web_ui")
|
|
21
21
|
VIEWS = File.join(ROOT, "views")
|
|
22
22
|
|
|
23
|
+
# Number of in-flight requests listed on the dashboard. The oldest are
|
|
24
|
+
# listed, because those are the ones worth looking at.
|
|
25
|
+
INFLIGHT_LIMIT = 50
|
|
26
|
+
|
|
23
27
|
class << self
|
|
24
28
|
# Sidekiq resolves symbol view names to different file names depending
|
|
25
29
|
# on the version (*.erb before 8.1, *.html.erb from 8.1 on), so the
|
|
@@ -28,6 +32,42 @@ module PatientHttp
|
|
|
28
32
|
@template ||= File.read(File.join(VIEWS, "patient_http.html.erb"))
|
|
29
33
|
end
|
|
30
34
|
|
|
35
|
+
# Build the per-processor rows for the dashboard by combining the
|
|
36
|
+
# cumulative counters with the live capacity each processor reports.
|
|
37
|
+
#
|
|
38
|
+
# Statistics are not recorded per processor when only one is
|
|
39
|
+
# configured, because they would duplicate the overall totals, so the
|
|
40
|
+
# breakdown is empty in that case.
|
|
41
|
+
#
|
|
42
|
+
# @param stats_by_processor [Hash, nil] cumulative counters by processor name
|
|
43
|
+
# @param capacity_by_processor [Hash] inflight and capacity by processor name
|
|
44
|
+
# @return [Array<Hash>] one row per processor, sorted by name
|
|
45
|
+
def processor_rows(stats_by_processor, capacity_by_processor)
|
|
46
|
+
stats_by_processor ||= {}
|
|
47
|
+
return [] if stats_by_processor.empty? && capacity_by_processor.size < 2
|
|
48
|
+
|
|
49
|
+
names = (stats_by_processor.keys | capacity_by_processor.keys).sort
|
|
50
|
+
names.map do |name|
|
|
51
|
+
counts = stats_by_processor[name] || {}
|
|
52
|
+
capacity = capacity_by_processor[name] || {}
|
|
53
|
+
requests = counts["requests"].to_i
|
|
54
|
+
max_capacity = capacity[:max_capacity].to_i
|
|
55
|
+
inflight = capacity[:inflight].to_i
|
|
56
|
+
|
|
57
|
+
{
|
|
58
|
+
name: name,
|
|
59
|
+
inflight: inflight,
|
|
60
|
+
peak_inflight: counts["max_inflight"].to_i,
|
|
61
|
+
max_capacity: max_capacity,
|
|
62
|
+
utilization: (max_capacity > 0) ? (inflight.to_f / max_capacity * 100).round(1) : 0,
|
|
63
|
+
requests: requests,
|
|
64
|
+
errors: counts["errors"].to_i,
|
|
65
|
+
max_capacity_exceeded: counts["max_capacity_exceeded"].to_i,
|
|
66
|
+
avg_duration: (requests > 0) ? (counts["duration"].to_f / requests).round(3) : 0.0
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
31
71
|
# This method is called by Sidekiq::Web when registering the extension
|
|
32
72
|
def registered(app)
|
|
33
73
|
# GET route for the main PatientHttp dashboard page
|
|
@@ -47,6 +87,16 @@ module PatientHttp
|
|
|
47
87
|
current_inflight = processes.values.sum { |data| data[:inflight] }
|
|
48
88
|
utilization = (max_capacity > 0) ? (current_inflight.to_f / max_capacity * 100).round(1) : 0
|
|
49
89
|
|
|
90
|
+
# Per-processor breakdown, combining the cumulative counters with
|
|
91
|
+
# the capacity each processor reports.
|
|
92
|
+
processors = PatientHttp::Sidekiq::WebUI.processor_rows(
|
|
93
|
+
totals["processors"],
|
|
94
|
+
PatientHttp::Sidekiq::TaskMonitor.inflight_counts_by_processor(processes)
|
|
95
|
+
)
|
|
96
|
+
|
|
97
|
+
# The requests that have been in flight the longest.
|
|
98
|
+
inflight = PatientHttp::Sidekiq::TaskMonitor.inflight_details(limit: INFLIGHT_LIMIT)
|
|
99
|
+
|
|
50
100
|
erb(PatientHttp::Sidekiq::WebUI.template, locals: {
|
|
51
101
|
totals: totals,
|
|
52
102
|
total_requests: total_requests,
|
|
@@ -54,7 +104,9 @@ module PatientHttp
|
|
|
54
104
|
max_capacity: max_capacity,
|
|
55
105
|
current_inflight: current_inflight,
|
|
56
106
|
utilization: utilization,
|
|
57
|
-
processes: processes
|
|
107
|
+
processes: processes,
|
|
108
|
+
processors: processors,
|
|
109
|
+
inflight: inflight
|
|
58
110
|
})
|
|
59
111
|
end
|
|
60
112
|
|
data/lib/patient_http/sidekiq.rb
CHANGED
|
@@ -79,6 +79,7 @@ module PatientHttp
|
|
|
79
79
|
autoload :Context, File.join(__dir__, "sidekiq/context")
|
|
80
80
|
autoload :DirectTaskHandler, File.join(__dir__, "sidekiq/direct_task_handler")
|
|
81
81
|
autoload :ProcessorObserver, File.join(__dir__, "sidekiq/processor_observer")
|
|
82
|
+
autoload :RedisPool, File.join(__dir__, "sidekiq/redis_pool")
|
|
82
83
|
autoload :RequestExecutor, File.join(__dir__, "sidekiq/request_executor")
|
|
83
84
|
autoload :RequestWorker, File.join(__dir__, "sidekiq/request_worker")
|
|
84
85
|
autoload :LifecycleHooks, File.join(__dir__, "sidekiq/lifecycle_hooks")
|
|
@@ -88,13 +89,17 @@ module PatientHttp
|
|
|
88
89
|
autoload :TaskMonitorThread, File.join(__dir__, "sidekiq/task_monitor_thread")
|
|
89
90
|
autoload :WebUI, File.join(__dir__, "sidekiq/web_ui")
|
|
90
91
|
|
|
91
|
-
@
|
|
92
|
+
@processors = {}
|
|
92
93
|
@configuration = nil
|
|
93
94
|
@after_completion_callbacks = []
|
|
94
95
|
@after_error_callbacks = []
|
|
95
96
|
@external_storage = nil
|
|
96
97
|
@request_handler = nil
|
|
97
98
|
@lifecycle_mutex = Mutex.new
|
|
99
|
+
@redis_pool = nil
|
|
100
|
+
@stats = nil
|
|
101
|
+
@task_monitor = nil
|
|
102
|
+
@monitor_thread = nil
|
|
98
103
|
|
|
99
104
|
class << self
|
|
100
105
|
attr_writer :configuration
|
|
@@ -111,6 +116,9 @@ module PatientHttp
|
|
|
111
116
|
yield(configuration) if block_given?
|
|
112
117
|
@configuration = configuration
|
|
113
118
|
@external_storage = nil
|
|
119
|
+
# Rebuild the stats aggregator from the new configuration unless a
|
|
120
|
+
# running processor already owns it.
|
|
121
|
+
@stats = nil unless running?
|
|
114
122
|
register_handler
|
|
115
123
|
PatientHttp.default_configuration = configuration
|
|
116
124
|
configuration
|
|
@@ -161,33 +169,33 @@ module PatientHttp
|
|
|
161
169
|
end
|
|
162
170
|
end
|
|
163
171
|
|
|
164
|
-
# Check if
|
|
172
|
+
# Check if any processor is running.
|
|
165
173
|
#
|
|
166
174
|
# @return [Boolean]
|
|
167
175
|
def running?
|
|
168
|
-
|
|
176
|
+
@processors.values.any?(&:running?)
|
|
169
177
|
end
|
|
170
178
|
|
|
171
|
-
# Check if
|
|
179
|
+
# Check if any processor is draining (not accepting new requests
|
|
172
180
|
# but still processing in-flight ones).
|
|
173
181
|
#
|
|
174
182
|
# @return [Boolean]
|
|
175
183
|
def draining?
|
|
176
|
-
|
|
184
|
+
@processors.values.any?(&:draining?)
|
|
177
185
|
end
|
|
178
186
|
|
|
179
|
-
# Check if
|
|
187
|
+
# Check if any processor is in the process of stopping.
|
|
180
188
|
#
|
|
181
189
|
# @return [Boolean]
|
|
182
190
|
def stopping?
|
|
183
|
-
|
|
191
|
+
@processors.values.any?(&:stopping?)
|
|
184
192
|
end
|
|
185
193
|
|
|
186
|
-
# Check if
|
|
194
|
+
# Check if all processors are stopped or none have been started.
|
|
187
195
|
#
|
|
188
196
|
# @return [Boolean]
|
|
189
197
|
def stopped?
|
|
190
|
-
@
|
|
198
|
+
@processors.values.all?(&:stopped?)
|
|
191
199
|
end
|
|
192
200
|
|
|
193
201
|
# Get an ExternalStorage instance for storing and fetching payloads.
|
|
@@ -256,8 +264,11 @@ module PatientHttp
|
|
|
256
264
|
# +error.callback_args+ using symbol or string keys.
|
|
257
265
|
# @param raise_error_responses [Boolean] If true, treats non-2xx responses as errors
|
|
258
266
|
# and calls +on_error+ instead of +on_complete+. Defaults to false.
|
|
267
|
+
# @param processor [Symbol, String, nil] Name of the processor profile that should
|
|
268
|
+
# execute the request. Defaults to the request's own processor name, a
|
|
269
|
+
# "processor" value from with_sidekiq_options, or :default.
|
|
259
270
|
# @return [String] the request ID
|
|
260
|
-
def execute(request, callback:, callback_args: nil, raise_error_responses: false)
|
|
271
|
+
def execute(request, callback:, callback_args: nil, raise_error_responses: false, processor: nil)
|
|
261
272
|
PatientHttp::CallbackValidator.validate!(callback)
|
|
262
273
|
callback_name = callback.is_a?(Class) ? callback.name : callback.to_s
|
|
263
274
|
callback_args = PatientHttp::CallbackValidator.validate_callback_args(callback_args)
|
|
@@ -273,11 +284,13 @@ module PatientHttp
|
|
|
273
284
|
end
|
|
274
285
|
|
|
275
286
|
options = current_sidekiq_options
|
|
287
|
+
processor_name = resolve_processor_name(processor, request, options)
|
|
276
288
|
if options&.any?
|
|
289
|
+
options = options.except("processor")
|
|
277
290
|
queue = options["queue"]
|
|
278
291
|
options = options.merge("patient_http_callback_queue" => queue.to_s) if queue
|
|
279
292
|
end
|
|
280
|
-
args = [data, callback_name, raise_error_responses, callback_args, request_id]
|
|
293
|
+
args = [data, callback_name, raise_error_responses, callback_args, request_id, processor_name]
|
|
281
294
|
|
|
282
295
|
if direct_execution?(options)
|
|
283
296
|
execute_on_local_processor(
|
|
@@ -286,7 +299,8 @@ module PatientHttp
|
|
|
286
299
|
callback_name: callback_name,
|
|
287
300
|
raise_error_responses: raise_error_responses,
|
|
288
301
|
callback_args: callback_args,
|
|
289
|
-
request_id: request_id
|
|
302
|
+
request_id: request_id,
|
|
303
|
+
processor_name: processor_name
|
|
290
304
|
)
|
|
291
305
|
elsif options&.any?
|
|
292
306
|
RequestWorker.set(options).perform_async(*args)
|
|
@@ -314,36 +328,62 @@ module PatientHttp
|
|
|
314
328
|
PatientHttp.register_handler(@request_handler)
|
|
315
329
|
end
|
|
316
330
|
|
|
317
|
-
# Start
|
|
331
|
+
# Start a processor for each configured processor profile, along with the
|
|
332
|
+
# shared crash-recovery monitor and stats.
|
|
318
333
|
#
|
|
319
334
|
# @return [void]
|
|
320
335
|
def start
|
|
321
336
|
@lifecycle_mutex.synchronize do
|
|
322
|
-
return if @
|
|
337
|
+
return if @processors.any? && !@processors.values.all?(&:stopped?)
|
|
323
338
|
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
339
|
+
warn_about_blocking_redis_driver
|
|
340
|
+
|
|
341
|
+
@redis_pool ||= RedisPool.new(configuration)
|
|
342
|
+
@stats ||= Stats.new(configuration)
|
|
343
|
+
@task_monitor ||= TaskMonitor.new(
|
|
344
|
+
configuration,
|
|
345
|
+
processors: -> { processor_capacity_snapshot }
|
|
346
|
+
)
|
|
347
|
+
|
|
348
|
+
@processors = {}
|
|
349
|
+
configuration.processor_profiles.each_key do |name|
|
|
350
|
+
processor = PatientHttp::Processor.new(configuration.processor_config(name), name: name)
|
|
351
|
+
processor.observe(ProcessorObserver.new(processor, stats: @stats, task_monitor: @task_monitor))
|
|
352
|
+
configuration.observers.each do |observer|
|
|
353
|
+
processor.observe(observer)
|
|
354
|
+
end
|
|
355
|
+
@processors[name] = processor
|
|
328
356
|
end
|
|
329
|
-
@
|
|
357
|
+
@processors.each_value(&:start)
|
|
358
|
+
|
|
359
|
+
# A restart after the processors stopped on their own (e.g. a reactor
|
|
360
|
+
# error) leaves the previous monitor thread running; stop it before
|
|
361
|
+
# replacing the reference so only one thread ever heartbeats.
|
|
362
|
+
@monitor_thread&.stop
|
|
363
|
+
@monitor_thread = TaskMonitorThread.new(
|
|
364
|
+
configuration,
|
|
365
|
+
@task_monitor,
|
|
366
|
+
-> { @processors.values.flat_map(&:tracked_request_ids) },
|
|
367
|
+
stats: @stats
|
|
368
|
+
)
|
|
369
|
+
@monitor_thread.start
|
|
330
370
|
end
|
|
331
371
|
|
|
332
372
|
register_handler
|
|
333
373
|
end
|
|
334
374
|
|
|
335
|
-
# Signal
|
|
375
|
+
# Signal all processors to drain (stop accepting new requests)
|
|
336
376
|
#
|
|
337
377
|
# @return [void]
|
|
338
378
|
def quiet
|
|
339
379
|
@lifecycle_mutex.synchronize do
|
|
340
380
|
return unless running?
|
|
341
381
|
|
|
342
|
-
@
|
|
382
|
+
@processors.each_value(&:drain)
|
|
343
383
|
end
|
|
344
384
|
end
|
|
345
385
|
|
|
346
|
-
# Stop
|
|
386
|
+
# Stop all processors gracefully
|
|
347
387
|
#
|
|
348
388
|
# @param timeout [Float, nil] maximum time to wait for in-flight requests to complete
|
|
349
389
|
# @return [void]
|
|
@@ -353,10 +393,12 @@ module PatientHttp
|
|
|
353
393
|
end
|
|
354
394
|
|
|
355
395
|
@lifecycle_mutex.synchronize do
|
|
356
|
-
|
|
396
|
+
# Shared services can outlive the processors when a start failed part
|
|
397
|
+
# way through, so tear them down whenever any of them exist.
|
|
398
|
+
return if @processors.empty? && @redis_pool.nil? && @task_monitor.nil? && @monitor_thread.nil?
|
|
357
399
|
|
|
358
|
-
|
|
359
|
-
|
|
400
|
+
stop_processors(timeout)
|
|
401
|
+
shutdown_shared_services
|
|
360
402
|
end
|
|
361
403
|
end
|
|
362
404
|
|
|
@@ -366,8 +408,8 @@ module PatientHttp
|
|
|
366
408
|
# @api private
|
|
367
409
|
def reset!
|
|
368
410
|
@lifecycle_mutex.synchronize do
|
|
369
|
-
|
|
370
|
-
|
|
411
|
+
stop_processors(0)
|
|
412
|
+
shutdown_shared_services
|
|
371
413
|
end
|
|
372
414
|
@configuration = nil
|
|
373
415
|
@external_storage = nil
|
|
@@ -398,14 +440,162 @@ module PatientHttp
|
|
|
398
440
|
end
|
|
399
441
|
end
|
|
400
442
|
|
|
401
|
-
# Returns
|
|
443
|
+
# Returns a processor instance by name (internal accessor).
|
|
402
444
|
#
|
|
445
|
+
# @param name [Symbol, String] the processor name
|
|
403
446
|
# @return [PatientHttp::Processor, nil]
|
|
404
447
|
# @api private
|
|
405
|
-
|
|
448
|
+
def processor(name = :default)
|
|
449
|
+
@processors[name.to_sym]
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# Set the default processor (internal, for testing).
|
|
453
|
+
#
|
|
454
|
+
# @param value [PatientHttp::Processor, nil]
|
|
455
|
+
# @api private
|
|
456
|
+
def processor=(value)
|
|
457
|
+
if value.nil?
|
|
458
|
+
@processors.delete(:default)
|
|
459
|
+
else
|
|
460
|
+
@processors[:default] = value
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
# The gem's dedicated Redis pool, or nil when no processor has started
|
|
465
|
+
# in this process (e.g. web or client processes).
|
|
466
|
+
#
|
|
467
|
+
# @return [RedisPool, nil]
|
|
468
|
+
# @api private
|
|
469
|
+
attr_reader :redis_pool
|
|
470
|
+
|
|
471
|
+
# The shared stats aggregator for this process. Available before start so
|
|
472
|
+
# rejection stats can be recorded from any path.
|
|
473
|
+
#
|
|
474
|
+
# @return [Stats]
|
|
475
|
+
# @api private
|
|
476
|
+
def stats
|
|
477
|
+
@stats ||= Stats.new(configuration)
|
|
478
|
+
end
|
|
479
|
+
|
|
480
|
+
# Yield a Redis connection: the gem's dedicated pool when it exists,
|
|
481
|
+
# falling back to Sidekiq's pool selection otherwise.
|
|
482
|
+
#
|
|
483
|
+
# @param retry_on_connection_error [Boolean] whether a connection-level
|
|
484
|
+
# failure may replay the block. Pass false when the block is not
|
|
485
|
+
# idempotent, such as a batch of counter increments the server may
|
|
486
|
+
# already have applied.
|
|
487
|
+
# @yield [conn] the Redis connection
|
|
488
|
+
# @return [Object] the block's return value
|
|
489
|
+
# @api private
|
|
490
|
+
def redis(retry_on_connection_error: true, &block)
|
|
491
|
+
pool = @redis_pool
|
|
492
|
+
if pool
|
|
493
|
+
pool.with(retry_on_connection_error: retry_on_connection_error, &block)
|
|
494
|
+
else
|
|
495
|
+
::Sidekiq.redis(&block)
|
|
496
|
+
end
|
|
497
|
+
end
|
|
498
|
+
|
|
499
|
+
# Run a block with Sidekiq client pushes routed through the gem's
|
|
500
|
+
# dedicated Redis pool. Used for pushes made from gem-owned threads
|
|
501
|
+
# (completion workers, the monitor thread) so they do not contend with
|
|
502
|
+
# Sidekiq's internal pool.
|
|
503
|
+
#
|
|
504
|
+
# @return [Object] the block's return value
|
|
505
|
+
# @api private
|
|
506
|
+
def with_redis_pool(&block)
|
|
507
|
+
pool = @redis_pool&.pool
|
|
508
|
+
if pool
|
|
509
|
+
::Sidekiq::Client.via(pool, &block)
|
|
510
|
+
else
|
|
511
|
+
yield
|
|
512
|
+
end
|
|
513
|
+
end
|
|
406
514
|
|
|
407
515
|
private
|
|
408
516
|
|
|
517
|
+
# Snapshot of each processor's capacity in this process, published with
|
|
518
|
+
# the heartbeat so the Web UI can report capacity per processor. The
|
|
519
|
+
# inflight count is the number of requests counted against the
|
|
520
|
+
# processor's capacity: queued, pending, and in flight.
|
|
521
|
+
#
|
|
522
|
+
# @return [Hash] processor name => { inflight:, max_capacity: }
|
|
523
|
+
def processor_capacity_snapshot
|
|
524
|
+
@processors.each_with_object({}) do |(name, processor), snapshot|
|
|
525
|
+
snapshot[name] = {
|
|
526
|
+
inflight: processor.total_count,
|
|
527
|
+
max_capacity: processor.config.max_connections
|
|
528
|
+
}
|
|
529
|
+
end
|
|
530
|
+
end
|
|
531
|
+
|
|
532
|
+
# Stop every processor and clear the registry. Each processor waits up to
|
|
533
|
+
# the full timeout for its in-flight requests, so they are stopped in
|
|
534
|
+
# parallel; stopping them in sequence would multiply the shutdown
|
|
535
|
+
# deadline by the number of processor profiles and overrun the time
|
|
536
|
+
# Sidekiq allows before it kills the process.
|
|
537
|
+
#
|
|
538
|
+
# @param timeout [Float, nil] maximum time to wait for in-flight requests
|
|
539
|
+
# @return [void]
|
|
540
|
+
def stop_processors(timeout)
|
|
541
|
+
processors = @processors.values
|
|
542
|
+
@processors = {}
|
|
543
|
+
return if processors.empty?
|
|
544
|
+
|
|
545
|
+
if processors.size == 1
|
|
546
|
+
processors.first.stop(timeout: timeout)
|
|
547
|
+
return
|
|
548
|
+
end
|
|
549
|
+
|
|
550
|
+
processors.map { |processor|
|
|
551
|
+
Thread.new do
|
|
552
|
+
processor.stop(timeout: timeout)
|
|
553
|
+
rescue => e
|
|
554
|
+
configuration.logger&.error(
|
|
555
|
+
"[PatientHttp::Sidekiq] Failed to stop processor #{processor.name}: #{e.inspect}"
|
|
556
|
+
)
|
|
557
|
+
end
|
|
558
|
+
}.each(&:join)
|
|
559
|
+
end
|
|
560
|
+
|
|
561
|
+
# Stop the shared monitor thread, flush pending stats, remove this
|
|
562
|
+
# process from the registry, and shut down the Redis pool. Called with
|
|
563
|
+
# the lifecycle mutex held after all processors have stopped.
|
|
564
|
+
def shutdown_shared_services
|
|
565
|
+
@monitor_thread&.stop
|
|
566
|
+
@monitor_thread = nil
|
|
567
|
+
begin
|
|
568
|
+
@stats&.flush
|
|
569
|
+
rescue => e
|
|
570
|
+
configuration.logger&.error("[PatientHttp::Sidekiq] Failed to flush stats during shutdown: #{e.inspect}")
|
|
571
|
+
end
|
|
572
|
+
begin
|
|
573
|
+
@task_monitor&.remove_process
|
|
574
|
+
rescue => e
|
|
575
|
+
configuration.logger&.error("[PatientHttp::Sidekiq] Failed to remove process registration: #{e.inspect}")
|
|
576
|
+
end
|
|
577
|
+
@task_monitor = nil
|
|
578
|
+
@stats = nil
|
|
579
|
+
@redis_pool&.shutdown
|
|
580
|
+
@redis_pool = nil
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
# Log a warning when a blocking C-level Redis driver is installed.
|
|
584
|
+
# Blocking Redis I/O on the reactor thread stalls every in-flight HTTP
|
|
585
|
+
# request; the gem's own calls run on worker threads, but application
|
|
586
|
+
# observers may still call Redis from processor callbacks.
|
|
587
|
+
def warn_about_blocking_redis_driver
|
|
588
|
+
return unless defined?(RedisClient) && RedisClient.default_driver.name.to_s.include?("Hiredis")
|
|
589
|
+
|
|
590
|
+
configuration.logger&.warn(
|
|
591
|
+
"[PatientHttp::Sidekiq] hiredis-client detected. The hiredis driver performs blocking I/O " \
|
|
592
|
+
"that does not yield to the fiber scheduler. Avoid Redis calls from processor observers " \
|
|
593
|
+
"or callbacks that run on the reactor thread."
|
|
594
|
+
)
|
|
595
|
+
rescue => e
|
|
596
|
+
configuration.logger&.debug("[PatientHttp::Sidekiq] Redis driver check failed: #{e.inspect}")
|
|
597
|
+
end
|
|
598
|
+
|
|
409
599
|
# Current scoped Sidekiq options, if any.
|
|
410
600
|
#
|
|
411
601
|
# @return [Hash, nil]
|
|
@@ -413,6 +603,19 @@ module PatientHttp
|
|
|
413
603
|
Thread.current[:patient_http_sidekiq_options]
|
|
414
604
|
end
|
|
415
605
|
|
|
606
|
+
# Resolve the processor profile name for a request. Precedence: the
|
|
607
|
+
# explicit +processor:+ argument, the request's own processor name, a
|
|
608
|
+
# "processor" value from with_sidekiq_options, then :default.
|
|
609
|
+
#
|
|
610
|
+
# @param explicit [Symbol, String, nil] the processor: argument
|
|
611
|
+
# @param request [PatientHttp::Request] the request
|
|
612
|
+
# @param options [Hash, nil] scoped Sidekiq options
|
|
613
|
+
# @return [String] the processor name
|
|
614
|
+
def resolve_processor_name(explicit, request, options)
|
|
615
|
+
name = explicit || request.processor || options&.[]("processor") || :default
|
|
616
|
+
name.to_s
|
|
617
|
+
end
|
|
618
|
+
|
|
416
619
|
# Check if the request can go directly to a processor running in the current
|
|
417
620
|
# process. Requests made in a with_sidekiq_options block always go through
|
|
418
621
|
# the queue so that Sidekiq applies the options (queue routing, scheduling,
|
|
@@ -440,8 +643,9 @@ module PatientHttp
|
|
|
440
643
|
# @param raise_error_responses [Boolean] whether non-2xx responses are errors
|
|
441
644
|
# @param callback_args [Hash, nil] arguments to pass to the callback
|
|
442
645
|
# @param request_id [String] unique request ID
|
|
646
|
+
# @param processor_name [String] name of the processor profile to run on
|
|
443
647
|
# @return [void]
|
|
444
|
-
def execute_on_local_processor(request_json, args, callback_name:, raise_error_responses:, callback_args:, request_id:)
|
|
648
|
+
def execute_on_local_processor(request_json, args, callback_name:, raise_error_responses:, callback_args:, request_id:, processor_name: "default")
|
|
445
649
|
task_handler = DirectTaskHandler.new(args)
|
|
446
650
|
|
|
447
651
|
begin
|
|
@@ -453,7 +657,8 @@ module PatientHttp
|
|
|
453
657
|
raise_error_responses: raise_error_responses,
|
|
454
658
|
callback_args: callback_args,
|
|
455
659
|
task_handler: task_handler,
|
|
456
|
-
request_id: request_id
|
|
660
|
+
request_id: request_id,
|
|
661
|
+
processor_name: processor_name
|
|
457
662
|
)
|
|
458
663
|
rescue PatientHttp::NotRunningError, PatientHttp::MaxCapacityError => e
|
|
459
664
|
configuration.logger&.info(
|