patient_http-solid_queue 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ARCHITECTURE.md +10 -8
- data/CHANGELOG.md +24 -0
- data/README.md +303 -234
- data/VERSION +1 -1
- data/lib/generators/patient_http/solid_queue/install_generator.rb +248 -0
- data/lib/generators/patient_http/solid_queue/templates/initializer.rb +102 -0
- data/lib/patient_http/solid_queue/callback_job.rb +14 -7
- data/lib/patient_http/solid_queue/configuration.rb +178 -76
- data/lib/patient_http/solid_queue/context.rb +10 -9
- data/lib/patient_http/solid_queue/engine.rb +3 -1
- data/lib/patient_http/solid_queue/gc_lock.rb +4 -4
- data/lib/patient_http/solid_queue/inflight_request.rb +4 -4
- data/lib/patient_http/solid_queue/lifecycle_hooks.rb +11 -1
- data/lib/patient_http/solid_queue/process_registration.rb +4 -4
- data/lib/patient_http/solid_queue/processor_observer.rb +43 -12
- data/lib/patient_http/solid_queue/record.rb +2 -1
- data/lib/patient_http/solid_queue/request_executor.rb +37 -20
- data/lib/patient_http/solid_queue/request_job.rb +21 -12
- data/lib/patient_http/solid_queue/task_handler.rb +44 -12
- data/lib/patient_http/solid_queue/task_monitor.rb +91 -73
- data/lib/patient_http/solid_queue/task_monitor_thread.rb +15 -13
- data/lib/patient_http/solid_queue.rb +274 -122
- data/patient_http-solid_queue.gemspec +2 -2
- metadata +8 -6
- /data/db/migrate/{20260216000000_create_solid_queue_async_http_tables.rb → 20260216000000_create_patient_http_solid_queue_tables.rb} +0 -0
|
@@ -4,34 +4,39 @@ require "delegate"
|
|
|
4
4
|
|
|
5
5
|
module PatientHttp
|
|
6
6
|
module SolidQueue
|
|
7
|
-
# Configuration for the Solid Queue
|
|
7
|
+
# Configuration for the Solid Queue integration.
|
|
8
8
|
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
9
|
+
# Extends `PatientHttp::Configuration` with Solid Queue defaults and adds
|
|
10
|
+
# options for the job queue, crash recovery, and named processor profiles.
|
|
11
11
|
class Configuration < PatientHttp::Configuration
|
|
12
|
-
# Default
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
#
|
|
16
|
-
|
|
12
|
+
# Default size in bytes above which payloads are stored externally.
|
|
13
|
+
#
|
|
14
|
+
# @deprecated Use {PatientHttp::Configuration::DEFAULT_PAYLOAD_STORE_THRESHOLD}.
|
|
15
|
+
# The `payload_store_threshold` option is defined on the base
|
|
16
|
+
# configuration, next to `register_payload_store`.
|
|
17
|
+
DEFAULT_PAYLOAD_STORE_THRESHOLD = PatientHttp::Configuration::DEFAULT_PAYLOAD_STORE_THRESHOLD
|
|
17
18
|
|
|
18
|
-
# @return [Numeric]
|
|
19
|
+
# @return [Numeric] The number of seconds without a heartbeat after which
|
|
20
|
+
# an in-flight request is considered orphaned and re-enqueued.
|
|
19
21
|
attr_reader :orphan_threshold
|
|
20
22
|
|
|
21
|
-
# @return [Numeric]
|
|
23
|
+
# @return [Numeric] The number of seconds between heartbeat updates for
|
|
24
|
+
# in-flight requests.
|
|
22
25
|
attr_reader :heartbeat_interval
|
|
23
26
|
|
|
24
|
-
# @return [String, nil]
|
|
27
|
+
# @return [String, nil] The queue name for `RequestJob` and `CallbackJob`.
|
|
25
28
|
attr_reader :queue_name
|
|
26
29
|
|
|
27
|
-
#
|
|
30
|
+
# Returns or sets the handler that runs when Active Job discards a
|
|
31
|
+
# `CallbackJob`.
|
|
32
|
+
#
|
|
28
33
|
# @overload on_retries_exhausted
|
|
29
34
|
# Returns the current handler.
|
|
30
|
-
# @return [#call, nil]
|
|
35
|
+
# @return [#call, nil] The handler, or `nil` if none is set.
|
|
31
36
|
# @overload on_retries_exhausted(&block)
|
|
32
37
|
# Sets a block as the handler.
|
|
33
|
-
# @yield [error] block to
|
|
34
|
-
# @yieldparam error [PatientHttp::Error]
|
|
38
|
+
# @yield [error] The block to run when a job is discarded.
|
|
39
|
+
# @yieldparam error [PatientHttp::Error] The error from the request.
|
|
35
40
|
def on_retries_exhausted(&block)
|
|
36
41
|
if block
|
|
37
42
|
@on_retries_exhausted = block
|
|
@@ -40,115 +45,199 @@ module PatientHttp
|
|
|
40
45
|
end
|
|
41
46
|
end
|
|
42
47
|
|
|
43
|
-
#
|
|
44
|
-
#
|
|
48
|
+
# Seconds subtracted from `SolidQueue.shutdown_timeout` to get the default
|
|
49
|
+
# `shutdown_timeout` for this gem's processor.
|
|
45
50
|
SHUTDOWN_TIMEOUT_BUFFER = 2
|
|
46
51
|
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
# @param
|
|
50
|
-
#
|
|
51
|
-
# @param
|
|
52
|
-
#
|
|
52
|
+
# Creates a configuration.
|
|
53
|
+
#
|
|
54
|
+
# @param heartbeat_interval [Numeric] The number of seconds between
|
|
55
|
+
# heartbeat updates for in-flight requests.
|
|
56
|
+
# @param orphan_threshold [Numeric] The number of seconds without a
|
|
57
|
+
# heartbeat after which an in-flight request is considered orphaned.
|
|
58
|
+
# @param queue_name [String, nil] The queue name for `RequestJob` and
|
|
59
|
+
# `CallbackJob`. If `nil`, the Active Job default queue applies.
|
|
60
|
+
# @param on_retries_exhausted [#call, nil] The handler that runs when
|
|
61
|
+
# Active Job discards a `CallbackJob`.
|
|
62
|
+
# @param pool_options [Hash] Options for `PatientHttp::Configuration`. If
|
|
63
|
+
# `shutdown_timeout` isn't set, it defaults to the Solid Queue shutdown
|
|
64
|
+
# timeout minus 2 seconds. If `logger` isn't set, it defaults to the
|
|
65
|
+
# Solid Queue logger.
|
|
66
|
+
# @raise [ArgumentError] If an option isn't valid.
|
|
53
67
|
def initialize(
|
|
54
68
|
heartbeat_interval: 60,
|
|
55
69
|
orphan_threshold: 300,
|
|
56
70
|
queue_name: nil,
|
|
57
|
-
payload_store_threshold: DEFAULT_PAYLOAD_STORE_THRESHOLD,
|
|
58
71
|
on_retries_exhausted: nil,
|
|
59
72
|
**pool_options
|
|
60
73
|
)
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
pool_options
|
|
65
|
-
pool_options[:logger] ||= (defined?(SolidQueue.logger) ? SolidQueue.logger : nil)
|
|
74
|
+
# The Solid Queue defaults for these options are read when the options
|
|
75
|
+
# are used, so settings that Solid Queue gets after this configuration
|
|
76
|
+
# is built still apply.
|
|
77
|
+
pool_options = pool_options.compact
|
|
66
78
|
|
|
67
79
|
super(**pool_options)
|
|
68
80
|
|
|
81
|
+
@shutdown_timeout_set = pool_options.key?(:shutdown_timeout)
|
|
82
|
+
@logger_set = pool_options.key?(:logger)
|
|
69
83
|
@processor_profiles = {default: {}}
|
|
84
|
+
@profile_configs = {}
|
|
85
|
+
@profile_configs_mutex = Mutex.new
|
|
70
86
|
self.queue_name = queue_name
|
|
71
87
|
self.heartbeat_interval = heartbeat_interval
|
|
72
88
|
self.orphan_threshold = orphan_threshold
|
|
73
|
-
self.payload_store_threshold = payload_store_threshold || DEFAULT_PAYLOAD_STORE_THRESHOLD
|
|
74
89
|
self.on_retries_exhausted = on_retries_exhausted
|
|
75
90
|
end
|
|
76
91
|
|
|
77
|
-
#
|
|
92
|
+
# Declares a named processor profile.
|
|
78
93
|
#
|
|
79
94
|
# Each profile becomes an independent processor with its own capacity,
|
|
80
|
-
# timeouts, and threads.
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
# processor
|
|
84
|
-
#
|
|
85
|
-
# +:default+ profile always exists; declaring it overrides options for
|
|
86
|
-
# the default processor.
|
|
95
|
+
# timeouts, and threads. The options override this configuration's HTTP
|
|
96
|
+
# options. With no options, the profile inherits every option. A request
|
|
97
|
+
# selects a processor with the `processor:` option, or with the
|
|
98
|
+
# request's own processor name. The `:default` profile always exists.
|
|
99
|
+
# Declare it to override options for the default processor.
|
|
87
100
|
#
|
|
88
101
|
# @example
|
|
89
|
-
# PatientHttp
|
|
102
|
+
# PatientHttp.configure do |config|
|
|
90
103
|
# config.processor(:llm, max_connections: 200, request_timeout: 120)
|
|
91
104
|
# config.processor(:webhooks, max_connections: 64, request_timeout: 10)
|
|
92
105
|
# end
|
|
93
106
|
#
|
|
94
|
-
# @param name [Symbol, String]
|
|
95
|
-
# @param options [Hash]
|
|
96
|
-
#
|
|
107
|
+
# @param name [Symbol, String] The processor name.
|
|
108
|
+
# @param options [Hash] Overrides for `PatientHttp::Configuration`
|
|
109
|
+
# options. `encryption_key` can't be overridden, because all processors
|
|
110
|
+
# share encryption.
|
|
111
|
+
# @return [Hash] The stored options for the profile.
|
|
112
|
+
# @raise [ArgumentError] If the name is empty or an option is invalid.
|
|
97
113
|
def processor(name, **options)
|
|
98
114
|
key = normalize_processor_name(name)
|
|
99
|
-
|
|
115
|
+
normalized = normalize_profile_options!(options)
|
|
116
|
+
@profile_configs_mutex.synchronize do
|
|
117
|
+
@profile_configs.delete(key)
|
|
118
|
+
@processor_profiles[key] = normalized
|
|
119
|
+
end
|
|
100
120
|
end
|
|
101
121
|
|
|
102
|
-
#
|
|
122
|
+
# Returns the options declared for a named processor profile.
|
|
103
123
|
#
|
|
104
|
-
# @param name [Symbol, String]
|
|
105
|
-
# @return [Hash, nil]
|
|
124
|
+
# @param name [Symbol, String] The processor name.
|
|
125
|
+
# @return [Hash, nil] The stored options, or `nil` if the profile isn't
|
|
126
|
+
# declared.
|
|
106
127
|
def processor_options(name)
|
|
107
|
-
|
|
128
|
+
key = name.to_s
|
|
129
|
+
return nil if key.empty?
|
|
130
|
+
|
|
131
|
+
@processor_profiles[key.to_sym]
|
|
108
132
|
end
|
|
109
133
|
|
|
110
|
-
#
|
|
134
|
+
# Returns all declared processor profiles, including `:default`.
|
|
111
135
|
#
|
|
112
|
-
# @return [Hash{Symbol => Hash}] profile options by processor
|
|
136
|
+
# @return [Hash{Symbol => Hash}] The profile options, keyed by processor
|
|
137
|
+
# name.
|
|
113
138
|
def processor_profiles
|
|
114
139
|
@processor_profiles.dup
|
|
115
140
|
end
|
|
116
141
|
|
|
117
|
-
#
|
|
118
|
-
# profile with no overrides is this configuration itself; other profiles
|
|
119
|
-
# get a view of this configuration with their overrides applied, so they
|
|
120
|
-
# share secrets, preprocessors, payload stores, and encryption.
|
|
142
|
+
# Returns the configuration for a named processor.
|
|
121
143
|
#
|
|
122
|
-
#
|
|
123
|
-
#
|
|
124
|
-
#
|
|
144
|
+
# A profile without overrides uses this configuration. Other profiles use
|
|
145
|
+
# a view of this configuration with their overrides applied, so all
|
|
146
|
+
# processors share secrets, preprocessors, payload stores, and
|
|
147
|
+
# encryption. The view is built once and reused until the profile is
|
|
148
|
+
# declared again.
|
|
149
|
+
#
|
|
150
|
+
# @param name [Symbol, String] The processor name.
|
|
151
|
+
# @return [PatientHttp::Configuration] The configuration for the processor.
|
|
152
|
+
# @raise [ArgumentError] If the profile isn't declared.
|
|
125
153
|
def processor_config(name)
|
|
126
154
|
key = normalize_processor_name(name)
|
|
127
|
-
profile = @processor_profiles[key]
|
|
128
|
-
raise ArgumentError.new("Unknown processor profile: #{name.inspect}") unless profile
|
|
129
155
|
|
|
130
|
-
|
|
156
|
+
@profile_configs_mutex.synchronize do
|
|
157
|
+
profile = @processor_profiles[key]
|
|
158
|
+
raise ArgumentError.new("Unknown processor profile: #{name.inspect}") unless profile
|
|
159
|
+
|
|
160
|
+
return self if profile.empty?
|
|
161
|
+
|
|
162
|
+
@profile_configs[key] ||= ProfileConfiguration.new(self, profile)
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
# Returns the graceful shutdown timeout in seconds. If it isn't set,
|
|
167
|
+
# returns the Solid Queue shutdown timeout minus 2 seconds, so that the
|
|
168
|
+
# processor stops before Solid Queue gives up on the worker.
|
|
169
|
+
#
|
|
170
|
+
# @return [Numeric] The timeout in seconds.
|
|
171
|
+
def shutdown_timeout
|
|
172
|
+
solid_queue_timeout = ::SolidQueue.shutdown_timeout
|
|
173
|
+
return super if @shutdown_timeout_set || solid_queue_timeout.nil?
|
|
174
|
+
|
|
175
|
+
[solid_queue_timeout - SHUTDOWN_TIMEOUT_BUFFER, 1].max
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# Sets the graceful shutdown timeout in seconds.
|
|
179
|
+
#
|
|
180
|
+
# @param value [Numeric] The timeout in seconds. Must be positive.
|
|
181
|
+
# @return [void]
|
|
182
|
+
# @raise [ArgumentError] If `value` isn't positive.
|
|
183
|
+
def shutdown_timeout=(value)
|
|
184
|
+
super
|
|
185
|
+
@shutdown_timeout_set = true
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Returns the logger. If it isn't set, returns the Solid Queue logger.
|
|
189
|
+
#
|
|
190
|
+
# @return [Logger] The logger.
|
|
191
|
+
def logger
|
|
192
|
+
return super if @logger_set
|
|
131
193
|
|
|
132
|
-
|
|
194
|
+
solid_queue_logger = ::SolidQueue.logger if ::SolidQueue.respond_to?(:logger)
|
|
195
|
+
solid_queue_logger || super
|
|
133
196
|
end
|
|
134
197
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
198
|
+
# Sets the logger.
|
|
199
|
+
#
|
|
200
|
+
# @param value [Logger, nil] The logger.
|
|
201
|
+
# @return [void]
|
|
202
|
+
def logger=(value)
|
|
203
|
+
super
|
|
204
|
+
@logger_set = true
|
|
138
205
|
end
|
|
139
206
|
|
|
207
|
+
# Sets the number of seconds between heartbeat updates for in-flight
|
|
208
|
+
# requests.
|
|
209
|
+
#
|
|
210
|
+
# @param value [Numeric] The interval in seconds. Must be positive and less
|
|
211
|
+
# than `orphan_threshold`.
|
|
212
|
+
# @return [void]
|
|
213
|
+
# @raise [ArgumentError] If `value` isn't positive or isn't less than
|
|
214
|
+
# `orphan_threshold`.
|
|
140
215
|
def heartbeat_interval=(value)
|
|
141
216
|
raise ArgumentError, "heartbeat_interval must be positive, got: #{value.inspect}" unless value.positive?
|
|
142
217
|
@heartbeat_interval = value
|
|
143
218
|
validate_heartbeat_and_threshold
|
|
144
219
|
end
|
|
145
220
|
|
|
221
|
+
# Sets the number of seconds without a heartbeat after which an in-flight
|
|
222
|
+
# request is considered orphaned and re-enqueued.
|
|
223
|
+
#
|
|
224
|
+
# @param value [Numeric] The threshold in seconds. Must be positive and
|
|
225
|
+
# greater than `heartbeat_interval`.
|
|
226
|
+
# @return [void]
|
|
227
|
+
# @raise [ArgumentError] If `value` isn't positive or isn't greater than
|
|
228
|
+
# `heartbeat_interval`.
|
|
146
229
|
def orphan_threshold=(value)
|
|
147
230
|
raise ArgumentError, "orphan_threshold must be positive, got: #{value.inspect}" unless value.positive?
|
|
148
231
|
@orphan_threshold = value
|
|
149
232
|
validate_heartbeat_and_threshold
|
|
150
233
|
end
|
|
151
234
|
|
|
235
|
+
# Sets the queue name for `RequestJob` and `CallbackJob`.
|
|
236
|
+
#
|
|
237
|
+
# @param name [String, nil] The queue name, or `nil` to use the Active Job
|
|
238
|
+
# default queue.
|
|
239
|
+
# @return [void]
|
|
240
|
+
# @raise [ArgumentError] If `name` isn't `nil` or a String.
|
|
152
241
|
def queue_name=(name)
|
|
153
242
|
if name.nil?
|
|
154
243
|
@queue_name = nil
|
|
@@ -160,13 +249,14 @@ module PatientHttp
|
|
|
160
249
|
apply_queue_name(name)
|
|
161
250
|
end
|
|
162
251
|
|
|
163
|
-
#
|
|
164
|
-
#
|
|
165
|
-
# This handler is called when a CallbackWorker job exhausts all retries.
|
|
166
|
-
# It receives the same arguments as the on_error callback.
|
|
252
|
+
# Sets the handler that runs when Active Job discards a `CallbackJob`. The
|
|
253
|
+
# handler receives the same error object as the `on_error` callback.
|
|
167
254
|
#
|
|
168
|
-
# @param value [#call, nil] A callable object or nil to
|
|
169
|
-
#
|
|
255
|
+
# @param value [#call, nil] A callable object, or `nil` to remove the
|
|
256
|
+
# handler.
|
|
257
|
+
# @return [void]
|
|
258
|
+
# @raise [ArgumentError] If `value` isn't `nil` and doesn't respond to
|
|
259
|
+
# `call`.
|
|
170
260
|
def on_retries_exhausted=(value)
|
|
171
261
|
if value && !value.respond_to?(:call)
|
|
172
262
|
raise ArgumentError.new("on_retries_exhausted must respond to #call, got: #{value.class}")
|
|
@@ -175,10 +265,12 @@ module PatientHttp
|
|
|
175
265
|
@on_retries_exhausted = value
|
|
176
266
|
end
|
|
177
267
|
|
|
178
|
-
#
|
|
268
|
+
# Returns the configuration as a Hash for inspection.
|
|
269
|
+
#
|
|
270
|
+
# @return [Hash{String => Object}] The option values, keyed by option
|
|
271
|
+
# name.
|
|
179
272
|
def to_h
|
|
180
273
|
super.merge(
|
|
181
|
-
"payload_store_threshold" => payload_store_threshold,
|
|
182
274
|
"heartbeat_interval" => heartbeat_interval,
|
|
183
275
|
"orphan_threshold" => orphan_threshold,
|
|
184
276
|
"queue_name" => queue_name,
|
|
@@ -187,11 +279,16 @@ module PatientHttp
|
|
|
187
279
|
)
|
|
188
280
|
end
|
|
189
281
|
|
|
190
|
-
#
|
|
191
|
-
#
|
|
192
|
-
#
|
|
193
|
-
# processors share
|
|
282
|
+
# A view of a base configuration with a processor profile's overrides
|
|
283
|
+
# applied. Options that the profile doesn't override, such as secrets,
|
|
284
|
+
# preprocessors, payload stores, and the logger, come from the base
|
|
285
|
+
# configuration, so all processors share them.
|
|
194
286
|
class ProfileConfiguration < SimpleDelegator
|
|
287
|
+
# Creates a view of the base configuration.
|
|
288
|
+
#
|
|
289
|
+
# @param base_configuration [PatientHttp::Configuration] The
|
|
290
|
+
# configuration to delegate to.
|
|
291
|
+
# @param overrides [Hash] Option values that replace the base values.
|
|
195
292
|
def initialize(base_configuration, overrides)
|
|
196
293
|
super(base_configuration)
|
|
197
294
|
overrides.each do |key, value|
|
|
@@ -207,13 +304,18 @@ module PatientHttp
|
|
|
207
304
|
PatientHttp::SolidQueue::CallbackJob.queue_as(name)
|
|
208
305
|
end
|
|
209
306
|
|
|
210
|
-
# Profile options must be valid PatientHttp::Configuration options
|
|
307
|
+
# Profile options must be valid PatientHttp::Configuration options other
|
|
308
|
+
# than `encryption_key`, which all processors share. A
|
|
211
309
|
# throwaway configuration exercises each option's own validation and
|
|
212
310
|
# normalization, so the stored value is what the writer would have
|
|
213
311
|
# produced rather than the raw input.
|
|
214
312
|
def normalize_profile_options!(options)
|
|
215
313
|
return options if options.empty?
|
|
216
314
|
|
|
315
|
+
if options.key?(:encryption_key)
|
|
316
|
+
raise ArgumentError.new("encryption_key can't be set for a processor profile")
|
|
317
|
+
end
|
|
318
|
+
|
|
217
319
|
probe = PatientHttp::Configuration.new(**options)
|
|
218
320
|
options.to_h do |key, value|
|
|
219
321
|
[key, probe.respond_to?(key) ? probe.public_send(key) : value]
|
|
@@ -2,26 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
#
|
|
5
|
+
# Stores the current Active Job for each thread.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# Code that runs in a job reads the job from this class instead of
|
|
8
|
+
# receiving it as an argument. `RequestJob` uses the job to re-enqueue a
|
|
9
|
+
# request.
|
|
10
10
|
class Context
|
|
11
11
|
@jobs = Concurrent::Map.new
|
|
12
12
|
|
|
13
13
|
class << self
|
|
14
|
-
# Returns the current
|
|
14
|
+
# Returns the current Active Job for this thread.
|
|
15
15
|
#
|
|
16
|
-
# @return [Hash, nil]
|
|
16
|
+
# @return [Hash, nil] The serialized job, or `nil` if no job is set.
|
|
17
17
|
def current_job
|
|
18
18
|
@jobs[Thread.current.object_id]
|
|
19
19
|
end
|
|
20
20
|
|
|
21
|
-
#
|
|
21
|
+
# Sets the current job for the duration of a block.
|
|
22
22
|
#
|
|
23
|
-
# @param job_data [Hash] Active Job
|
|
24
|
-
# @yield
|
|
23
|
+
# @param job_data [Hash] The serialized Active Job.
|
|
24
|
+
# @yield The block to run.
|
|
25
|
+
# @return [Object] The return value of the block.
|
|
25
26
|
def with_job(job_data)
|
|
26
27
|
thread_id = Thread.current.object_id
|
|
27
28
|
previous_job = @jobs[thread_id]
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Rails
|
|
5
|
+
# Rails engine that exposes the gem's migrations to the
|
|
6
|
+
# `patient_http_solid_queue:install:migrations` task. The migrations don't
|
|
7
|
+
# run with the host application's `db:migrate` until they are copied.
|
|
6
8
|
class Engine < ::Rails::Engine
|
|
7
9
|
engine_name "patient_http_solid_queue"
|
|
8
10
|
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Active Record model for distributed garbage collection
|
|
5
|
+
# Active Record model for the distributed garbage collection lock.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
# column records when
|
|
9
|
-
#
|
|
7
|
+
# The lock makes sure that only one process at a time runs orphan
|
|
8
|
+
# detection. The `last_gc_at` column records when garbage collection last
|
|
9
|
+
# completed, so a process can skip it if another process ran it recently.
|
|
10
10
|
class GcLock < Record
|
|
11
11
|
self.table_name = "patient_http_solid_queue_gc_locks"
|
|
12
12
|
end
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Active Record model
|
|
5
|
+
# Active Record model for in-flight HTTP requests, used for crash recovery.
|
|
6
6
|
#
|
|
7
|
-
# Each record represents
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# Each record represents one in-flight HTTP request. The processor updates
|
|
8
|
+
# the `heartbeat_at` timestamp periodically. Garbage collection finds stale
|
|
9
|
+
# records from dead processes and re-enqueues their requests.
|
|
10
10
|
class InflightRequest < Record
|
|
11
11
|
self.table_name = "patient_http_solid_queue_inflight_requests"
|
|
12
12
|
end
|
|
@@ -2,11 +2,21 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Registers lifecycle hooks
|
|
5
|
+
# Registers Solid Queue worker lifecycle hooks that manage the processors.
|
|
6
|
+
#
|
|
7
|
+
# The hooks do the following:
|
|
8
|
+
#
|
|
9
|
+
# - Start the processors when a Solid Queue worker starts
|
|
10
|
+
# (`on_worker_start`).
|
|
11
|
+
# - Stop the processors when the worker stops (`on_worker_stop`).
|
|
6
12
|
class LifecycleHooks
|
|
7
13
|
@registered = false
|
|
8
14
|
|
|
9
15
|
class << self
|
|
16
|
+
# Registers the lifecycle hooks. The gem calls this method when it loads.
|
|
17
|
+
# Repeated calls have no effect.
|
|
18
|
+
#
|
|
19
|
+
# @return [void]
|
|
10
20
|
def register
|
|
11
21
|
return if @registered
|
|
12
22
|
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Active Record model
|
|
5
|
+
# Active Record model for processes that run an async HTTP processor.
|
|
6
6
|
#
|
|
7
|
-
# Each record represents a running
|
|
8
|
-
# timestamp
|
|
9
|
-
# in this table are considered orphaned
|
|
7
|
+
# Each record represents a running process. Heartbeats update the
|
|
8
|
+
# `last_seen_at` timestamp. During garbage collection, in-flight requests
|
|
9
|
+
# from processes that aren't in this table are considered orphaned.
|
|
10
10
|
class ProcessRegistration < Record
|
|
11
11
|
self.table_name = "patient_http_solid_queue_processes"
|
|
12
12
|
end
|
|
@@ -2,22 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Processor observer that maintains the crash
|
|
6
|
-
# processor.
|
|
7
|
-
#
|
|
5
|
+
# Processor observer that maintains the crash recovery registry for one
|
|
6
|
+
# processor. All processors in the process share one task monitor, which
|
|
7
|
+
# the `PatientHttp::SolidQueue` module owns along with the monitor thread.
|
|
8
8
|
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
#
|
|
15
|
-
#
|
|
16
|
-
#
|
|
17
|
-
# instead of losing it.
|
|
9
|
+
# The observer registers a task when the processor accepts it, before
|
|
10
|
+
# `Processor#enqueue` returns. A request therefore has a durable record
|
|
11
|
+
# from the moment the caller hands it off. Registration runs on the
|
|
12
|
+
# caller's job worker thread, not on the reactor thread.
|
|
13
|
+
#
|
|
14
|
+
# The observer removes the entry when the request completes, or when an
|
|
15
|
+
# Active Job owns the request again because the task was rejected or
|
|
16
|
+
# re-enqueued. If result delivery fails, the observer keeps the entry, so
|
|
17
|
+
# the orphan collector re-enqueues the request instead of losing it.
|
|
18
18
|
class ProcessorObserver < PatientHttp::ProcessorObserver
|
|
19
|
+
# @return [TaskMonitor] The in-flight request registry.
|
|
19
20
|
attr_reader :task_monitor
|
|
20
21
|
|
|
22
|
+
# Creates an observer for a processor.
|
|
23
|
+
#
|
|
24
|
+
# @param processor [PatientHttp::Processor] The processor to observe.
|
|
25
|
+
# @param task_monitor [TaskMonitor] The in-flight request registry.
|
|
21
26
|
def initialize(processor, task_monitor:)
|
|
22
27
|
@processor = processor
|
|
23
28
|
@task_monitor = task_monitor
|
|
@@ -25,14 +30,29 @@ module PatientHttp
|
|
|
25
30
|
@requeued_mutex = Mutex.new
|
|
26
31
|
end
|
|
27
32
|
|
|
33
|
+
# Adds a request to the crash-recovery registry.
|
|
34
|
+
#
|
|
35
|
+
# @param request_task [PatientHttp::RequestTask] The request task.
|
|
36
|
+
# @return [void]
|
|
37
|
+
# @raise [RegistrationError] If the registry entry can't be written.
|
|
28
38
|
def request_enqueued(request_task)
|
|
29
39
|
task_monitor.register(request_task)
|
|
30
40
|
end
|
|
31
41
|
|
|
42
|
+
# Removes a rejected request from the crash-recovery registry. An Active
|
|
43
|
+
# Job owns the request again.
|
|
44
|
+
#
|
|
45
|
+
# @param request_task [PatientHttp::RequestTask] The request task.
|
|
46
|
+
# @return [void]
|
|
32
47
|
def request_rejected(request_task)
|
|
33
48
|
task_monitor.unregister(request_task)
|
|
34
49
|
end
|
|
35
50
|
|
|
51
|
+
# Removes a re-enqueued request from the crash-recovery registry. An
|
|
52
|
+
# Active Job owns the request again.
|
|
53
|
+
#
|
|
54
|
+
# @param request_task [PatientHttp::RequestTask] The request task.
|
|
55
|
+
# @return [void]
|
|
36
56
|
def request_requeued(request_task)
|
|
37
57
|
task_monitor.unregister(request_task)
|
|
38
58
|
# The re-enqueue path fires request_end after request_requeued, but
|
|
@@ -45,6 +65,10 @@ module PatientHttp
|
|
|
45
65
|
@requeued_mutex.synchronize { @requeued_task_ids << request_task.id }
|
|
46
66
|
end
|
|
47
67
|
|
|
68
|
+
# Removes a finished request from the crash-recovery registry.
|
|
69
|
+
#
|
|
70
|
+
# @param request_task [PatientHttp::RequestTask] The request task.
|
|
71
|
+
# @return [void]
|
|
48
72
|
def request_end(request_task)
|
|
49
73
|
requeued = @requeued_mutex.synchronize { @requeued_task_ids.delete?(request_task.id) }
|
|
50
74
|
return if requeued
|
|
@@ -52,6 +76,13 @@ module PatientHttp
|
|
|
52
76
|
task_monitor.unregister(request_task)
|
|
53
77
|
end
|
|
54
78
|
|
|
79
|
+
# Handles a failure to deliver a request's result. Keeps the
|
|
80
|
+
# crash-recovery record and releases it, so that the orphan collector
|
|
81
|
+
# re-enqueues the request, and logs the error.
|
|
82
|
+
#
|
|
83
|
+
# @param request_task [PatientHttp::RequestTask] The request task.
|
|
84
|
+
# @param error [Exception] The delivery failure.
|
|
85
|
+
# @return [void]
|
|
55
86
|
def completion_failed(request_task, error)
|
|
56
87
|
# Keep the crash-recovery registry entry, but hand it off to the orphan
|
|
57
88
|
# collector. Orphan collection ignores records that belong to a live
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Base Active Record class for
|
|
5
|
+
# Base Active Record class for this gem's models. The models use the
|
|
6
|
+
# Solid Queue database.
|
|
6
7
|
class Record < ::SolidQueue::Record
|
|
7
8
|
self.abstract_class = true
|
|
8
9
|
end
|