patient_http-solid_queue 1.2.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ARCHITECTURE.md +10 -8
- data/CHANGELOG.md +24 -0
- data/README.md +303 -234
- data/VERSION +1 -1
- data/lib/generators/patient_http/solid_queue/install_generator.rb +248 -0
- data/lib/generators/patient_http/solid_queue/templates/initializer.rb +102 -0
- data/lib/patient_http/solid_queue/callback_job.rb +14 -7
- data/lib/patient_http/solid_queue/configuration.rb +178 -76
- data/lib/patient_http/solid_queue/context.rb +10 -9
- data/lib/patient_http/solid_queue/engine.rb +3 -1
- data/lib/patient_http/solid_queue/gc_lock.rb +4 -4
- data/lib/patient_http/solid_queue/inflight_request.rb +4 -4
- data/lib/patient_http/solid_queue/lifecycle_hooks.rb +11 -1
- data/lib/patient_http/solid_queue/process_registration.rb +4 -4
- data/lib/patient_http/solid_queue/processor_observer.rb +43 -12
- data/lib/patient_http/solid_queue/record.rb +2 -1
- data/lib/patient_http/solid_queue/request_executor.rb +37 -20
- data/lib/patient_http/solid_queue/request_job.rb +21 -12
- data/lib/patient_http/solid_queue/task_handler.rb +44 -12
- data/lib/patient_http/solid_queue/task_monitor.rb +91 -73
- data/lib/patient_http/solid_queue/task_monitor_thread.rb +15 -13
- data/lib/patient_http/solid_queue.rb +274 -122
- data/patient_http-solid_queue.gemspec +2 -2
- metadata +8 -6
- /data/db/migrate/{20260216000000_create_solid_queue_async_http_tables.rb → 20260216000000_create_patient_http_solid_queue_tables.rb} +0 -0
|
@@ -2,21 +2,32 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
#
|
|
5
|
+
# Runs HTTP requests on a processor in the current process.
|
|
6
6
|
class RequestExecutor
|
|
7
7
|
class << self
|
|
8
|
-
#
|
|
8
|
+
# Hands the request to the async processor.
|
|
9
9
|
#
|
|
10
|
-
# @param request [PatientHttp::Request]
|
|
11
|
-
# @param callback [Class, String]
|
|
12
|
-
#
|
|
13
|
-
# @param
|
|
14
|
-
#
|
|
15
|
-
# @param
|
|
16
|
-
#
|
|
17
|
-
# @param
|
|
18
|
-
#
|
|
19
|
-
#
|
|
10
|
+
# @param request [PatientHttp::Request] The HTTP request to execute.
|
|
11
|
+
# @param callback [Class, String] The callback service class, or its
|
|
12
|
+
# fully qualified class name.
|
|
13
|
+
# @param active_job_data [Hash, nil] The serialized Active Job, with
|
|
14
|
+
# `"job_class"` and `"arguments"` keys. Defaults to the current job.
|
|
15
|
+
# @param synchronous [Boolean] If `true`, runs the request inline. Use
|
|
16
|
+
# this in tests.
|
|
17
|
+
# @param callback_args [#to_h, nil] Arguments to pass to the callback.
|
|
18
|
+
# @param raise_error_responses [Boolean, nil] If `true`, treats non-2xx
|
|
19
|
+
# responses as errors. If `nil`, uses the processor profile's
|
|
20
|
+
# `raise_error_responses` option.
|
|
21
|
+
# @param request_id [String, nil] A unique request ID for tracking.
|
|
22
|
+
# @param processor_name [Symbol, String, nil] The name of the processor
|
|
23
|
+
# profile that runs the request. Defaults to the request's processor
|
|
24
|
+
# name, or `:default`.
|
|
25
|
+
# @return [String] The request ID.
|
|
26
|
+
# @raise [ArgumentError] If the Active Job data is missing or invalid.
|
|
27
|
+
# @raise [PatientHttp::UnknownProcessorError] If the processor profile
|
|
28
|
+
# isn't configured.
|
|
29
|
+
# @raise [PatientHttp::NotRunningError] If the processor isn't running.
|
|
30
|
+
# @raise [PatientHttp::MaxCapacityError] If the processor is at capacity.
|
|
20
31
|
# @api private
|
|
21
32
|
def execute(
|
|
22
33
|
request,
|
|
@@ -24,19 +35,26 @@ module PatientHttp
|
|
|
24
35
|
active_job_data: nil,
|
|
25
36
|
synchronous: false,
|
|
26
37
|
callback_args: nil,
|
|
27
|
-
raise_error_responses:
|
|
38
|
+
raise_error_responses: nil,
|
|
28
39
|
request_id: nil,
|
|
29
40
|
processor_name: nil
|
|
30
41
|
)
|
|
31
42
|
active_job_data = validate_active_job_data(active_job_data)
|
|
32
|
-
task_handler = TaskHandler.new(active_job_data)
|
|
33
|
-
config = PatientHttp::SolidQueue.configuration
|
|
34
43
|
|
|
35
44
|
# Resolve the processor profile up front so the task is built with
|
|
36
|
-
# the options of the processor that will run it.
|
|
45
|
+
# the options of the processor that will run it. A running processor
|
|
46
|
+
# already holds its built profile configuration. An unknown name
|
|
37
47
|
# falls back to the base configuration here and is reported below.
|
|
38
48
|
name = (processor_name || request.processor || :default).to_sym
|
|
39
|
-
|
|
49
|
+
processor = PatientHttp::SolidQueue.processor(name)
|
|
50
|
+
declared_config = PatientHttp::SolidQueue.processor_config_for(name)
|
|
51
|
+
profile_config = declared_config || PatientHttp::SolidQueue.configuration
|
|
52
|
+
|
|
53
|
+
# A nil value means the caller did not ask for a specific behavior.
|
|
54
|
+
# Jobs enqueued by earlier versions of the gem can also carry nil.
|
|
55
|
+
raise_error_responses = profile_config.raise_error_responses if raise_error_responses.nil?
|
|
56
|
+
|
|
57
|
+
task_handler = TaskHandler.new(active_job_data, config: profile_config)
|
|
40
58
|
|
|
41
59
|
task = PatientHttp::RequestTask.new(
|
|
42
60
|
request: request,
|
|
@@ -58,12 +76,11 @@ module PatientHttp
|
|
|
58
76
|
return task.id
|
|
59
77
|
end
|
|
60
78
|
|
|
61
|
-
#
|
|
79
|
+
# An unknown processor name raises so the job
|
|
62
80
|
# lands in Active Job's retry mechanism instead of being dropped;
|
|
63
81
|
# this covers rolling deploys where an old process has not
|
|
64
82
|
# configured a new profile yet.
|
|
65
|
-
|
|
66
|
-
if processor.nil? && !config.processor_profiles.key?(name)
|
|
83
|
+
if declared_config.nil?
|
|
67
84
|
raise PatientHttp::UnknownProcessorError, "No processor profile configured for #{name.inspect}"
|
|
68
85
|
end
|
|
69
86
|
|
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Active Job that
|
|
5
|
+
# Active Job that hands HTTP requests to the async processor.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# `PatientHttp.get`, `PatientHttp.post`, and the other request methods
|
|
8
|
+
# enqueue this job. When the request completes, `CallbackJob` calls the
|
|
9
|
+
# callback service's `on_complete` or `on_error` method.
|
|
10
10
|
#
|
|
11
11
|
# @api private
|
|
12
12
|
class RequestJob < ActiveJob::Base
|
|
@@ -38,13 +38,22 @@ module PatientHttp
|
|
|
38
38
|
)
|
|
39
39
|
end
|
|
40
40
|
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
# @param
|
|
44
|
-
#
|
|
45
|
-
# @param
|
|
46
|
-
#
|
|
47
|
-
#
|
|
41
|
+
# Runs the HTTP request on a processor.
|
|
42
|
+
#
|
|
43
|
+
# @param data [Hash] The serialized request, or a reference to it in
|
|
44
|
+
# external storage. The request can be encrypted.
|
|
45
|
+
# @param callback_service_name [String] The fully qualified callback
|
|
46
|
+
# service class name.
|
|
47
|
+
# @param raise_error_responses [Boolean, nil] Whether to treat non-2xx
|
|
48
|
+
# responses as errors. If `nil`, uses the processor profile's
|
|
49
|
+
# `raise_error_responses` option.
|
|
50
|
+
# @param callback_args [Hash, nil] The arguments to pass to the callback.
|
|
51
|
+
# @param request_id [String, nil] The request ID.
|
|
52
|
+
# @param processor_name [String, nil] The name of the processor profile
|
|
53
|
+
# that runs the request. If `nil`, uses the processor set on the
|
|
54
|
+
# request, then the default processor. Jobs enqueued by earlier
|
|
55
|
+
# versions of the gem don't have this argument.
|
|
56
|
+
# @return [void]
|
|
48
57
|
def perform(data, callback_service_name, raise_error_responses, callback_args, request_id, processor_name = nil)
|
|
49
58
|
actual_data = PatientHttp::ExternalStorage.storage_ref?(data) ? PatientHttp::SolidQueue.external_storage.fetch(data) : data
|
|
50
59
|
actual_data = PatientHttp::SolidQueue.decrypt(actual_data)
|
|
@@ -63,7 +72,7 @@ module PatientHttp
|
|
|
63
72
|
callback_args: callback_args,
|
|
64
73
|
active_job_data: active_job_data,
|
|
65
74
|
request_id: request_id,
|
|
66
|
-
processor_name: processor_name
|
|
75
|
+
processor_name: processor_name
|
|
67
76
|
)
|
|
68
77
|
end
|
|
69
78
|
end
|
|
@@ -2,50 +2,82 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Active Job
|
|
5
|
+
# Task handler that uses Active Job to deliver results and retry requests.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
# -
|
|
9
|
-
#
|
|
10
|
-
# -
|
|
7
|
+
# - A `CallbackJob` delivers each result to the callback service.
|
|
8
|
+
# - Large payloads are written to external storage before the job is
|
|
9
|
+
# enqueued.
|
|
10
|
+
# - A retry enqueues the original Active Job again.
|
|
11
11
|
class TaskHandler < PatientHttp::TaskHandler
|
|
12
|
+
# @return [Hash] The serialized Active Job that made the request.
|
|
12
13
|
attr_reader :active_job_data
|
|
13
14
|
|
|
14
|
-
|
|
15
|
+
# Creates a task handler for an Active Job.
|
|
16
|
+
#
|
|
17
|
+
# @param active_job_data [Hash] The serialized Active Job that made the
|
|
18
|
+
# request.
|
|
19
|
+
# @param config [PatientHttp::Configuration, nil] The configuration of the
|
|
20
|
+
# processor that runs the request. If `nil`, uses the base configuration.
|
|
21
|
+
def initialize(active_job_data, config: nil)
|
|
15
22
|
@active_job_data = active_job_data
|
|
23
|
+
@config = config
|
|
16
24
|
end
|
|
17
25
|
|
|
26
|
+
# Enqueues a `CallbackJob` that calls the callback service's `on_complete`
|
|
27
|
+
# method. A large response is written to external storage first.
|
|
28
|
+
#
|
|
29
|
+
# @param response [PatientHttp::Response] The HTTP response.
|
|
30
|
+
# @param callback [String] The callback service class name.
|
|
31
|
+
# @return [void]
|
|
18
32
|
def on_complete(response, callback)
|
|
19
33
|
data = store_if_needed(response.as_json)
|
|
20
34
|
CallbackJob.perform_later(data, "response", callback)
|
|
21
35
|
delete_stored_request_payload
|
|
22
36
|
end
|
|
23
37
|
|
|
38
|
+
# Enqueues a `CallbackJob` that calls the callback service's `on_error`
|
|
39
|
+
# method. A large error is written to external storage first.
|
|
40
|
+
#
|
|
41
|
+
# @param error [PatientHttp::Error] The error.
|
|
42
|
+
# @param callback [String] The callback service class name.
|
|
43
|
+
# @return [void]
|
|
24
44
|
def on_error(error, callback)
|
|
25
45
|
data = store_if_needed(error.as_json)
|
|
26
46
|
CallbackJob.perform_later(data, "error", callback)
|
|
27
47
|
delete_stored_request_payload
|
|
28
48
|
end
|
|
29
49
|
|
|
50
|
+
# Re-enqueues the original Active Job with its execution count reset.
|
|
51
|
+
#
|
|
52
|
+
# @return [ActiveJob::Base, false] The enqueued job, or `false` if it
|
|
53
|
+
# wasn't enqueued.
|
|
30
54
|
def retry
|
|
31
55
|
ActiveJob::Base.deserialize(@active_job_data).tap { |j| j.executions = 0 }.enqueue
|
|
32
56
|
end
|
|
33
57
|
|
|
58
|
+
# Returns the Active Job ID.
|
|
59
|
+
#
|
|
60
|
+
# @return [String] The job ID.
|
|
34
61
|
def job_id
|
|
35
62
|
@active_job_data["job_id"]
|
|
36
63
|
end
|
|
37
64
|
|
|
65
|
+
# Returns the class of the Active Job.
|
|
66
|
+
#
|
|
67
|
+
# @return [Class] The job class.
|
|
38
68
|
def worker_class
|
|
39
69
|
PatientHttp::ClassHelper.resolve_class_name(@active_job_data["job_class"])
|
|
40
70
|
end
|
|
41
71
|
|
|
42
72
|
private
|
|
43
73
|
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
74
|
+
# Deletes the externally stored request payload after the request
|
|
75
|
+
# finishes. The payload must stay available until then, because Active Job
|
|
76
|
+
# retries, processor shutdown retries, and crash recovery can enqueue the
|
|
77
|
+
# job again. Applies only to `RequestJob` jobs, because other job types
|
|
78
|
+
# manage their own arguments.
|
|
79
|
+
#
|
|
80
|
+
# @return [void]
|
|
49
81
|
def delete_stored_request_payload
|
|
50
82
|
return unless @active_job_data["job_class"] == RequestJob.name
|
|
51
83
|
|
|
@@ -63,7 +95,7 @@ module PatientHttp
|
|
|
63
95
|
encrypted = PatientHttp::SolidQueue.encrypt(data)
|
|
64
96
|
external_storage = PatientHttp::SolidQueue.external_storage
|
|
65
97
|
if external_storage.enabled?
|
|
66
|
-
external_storage.store(encrypted, max_size: PatientHttp::SolidQueue.configuration.payload_store_threshold)
|
|
98
|
+
external_storage.store(encrypted, max_size: (@config || PatientHttp::SolidQueue.configuration).payload_store_threshold)
|
|
67
99
|
else
|
|
68
100
|
encrypted
|
|
69
101
|
end
|
|
@@ -2,27 +2,35 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
#
|
|
5
|
+
# Tracks in-flight requests in the database for crash recovery.
|
|
6
6
|
#
|
|
7
|
-
#
|
|
8
|
-
#
|
|
9
|
-
#
|
|
7
|
+
# The registry keeps an Active Record row for each in-flight request, with
|
|
8
|
+
# its heartbeat time and its Active Job. If a process crashes, another
|
|
9
|
+
# process finds the orphaned requests and re-enqueues their jobs. A
|
|
10
|
+
# distributed lock lets only one process at a time look for orphaned
|
|
11
|
+
# requests.
|
|
10
12
|
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# -
|
|
15
|
-
# -
|
|
13
|
+
# Each entry has a registry ID in the format
|
|
14
|
+
# `hostname:pid:hex/request-uuid`:
|
|
15
|
+
#
|
|
16
|
+
# - `hostname`: The host name, with colons and slashes replaced by dashes.
|
|
17
|
+
# - `pid`: The process ID.
|
|
18
|
+
# - `hex`: 16 random hex characters that make the ID unique.
|
|
19
|
+
# - `request-uuid`: The request ID.
|
|
16
20
|
class TaskMonitor
|
|
21
|
+
# Name of the garbage collection lock row.
|
|
17
22
|
GC_LOCK_NAME = "gc"
|
|
18
23
|
|
|
19
|
-
# @return [Configuration]
|
|
24
|
+
# @return [Configuration] The gem configuration.
|
|
20
25
|
attr_reader :config
|
|
21
26
|
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
25
|
-
#
|
|
27
|
+
# Creates a task monitor for this process.
|
|
28
|
+
#
|
|
29
|
+
# @param config [Configuration] The gem configuration.
|
|
30
|
+
# @param max_connections [#call, nil] A callable that returns the total
|
|
31
|
+
# maximum number of connections for the process. Defaults to the
|
|
32
|
+
# configuration's value. With named processors, the module passes the
|
|
33
|
+
# sum across all processors.
|
|
26
34
|
def initialize(config, max_connections: nil)
|
|
27
35
|
@config = config
|
|
28
36
|
@max_connections_source = max_connections || -> { config.max_connections }
|
|
@@ -31,16 +39,17 @@ module PatientHttp
|
|
|
31
39
|
@lock_identifier = "#{hostname}:#{pid}:#{SecureRandom.hex(8)}".freeze
|
|
32
40
|
end
|
|
33
41
|
|
|
34
|
-
#
|
|
42
|
+
# Records a request as in flight in the database.
|
|
35
43
|
#
|
|
36
|
-
# Runs on the caller's thread
|
|
37
|
-
# Errors propagate so a task is never accepted without a durable
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
# failing
|
|
44
|
+
# Runs on the caller's thread through the `request_enqueued` observer
|
|
45
|
+
# event. Errors propagate, so a task is never accepted without a durable
|
|
46
|
+
# record. The processor rejects the task, and the enqueue raises to the
|
|
47
|
+
# caller. The error is wrapped in `RegistrationError` so the job retries
|
|
48
|
+
# instead of failing, because the usual cause is a transient database
|
|
49
|
+
# issue.
|
|
41
50
|
#
|
|
42
|
-
# @param task [PatientHttp::RequestTask]
|
|
43
|
-
# @raise [RegistrationError]
|
|
51
|
+
# @param task [PatientHttp::RequestTask] The request task to register.
|
|
52
|
+
# @raise [RegistrationError] If the record can't be written.
|
|
44
53
|
# @return [void]
|
|
45
54
|
def register(task)
|
|
46
55
|
job_payload = task.task_handler.active_job_data.to_json
|
|
@@ -61,9 +70,9 @@ module PatientHttp
|
|
|
61
70
|
raise RegistrationError.new("Failed to register task #{task_id}: #{e.class} - #{e.message}")
|
|
62
71
|
end
|
|
63
72
|
|
|
64
|
-
#
|
|
73
|
+
# Removes a request from the registry.
|
|
65
74
|
#
|
|
66
|
-
# @param task [PatientHttp::RequestTask]
|
|
75
|
+
# @param task [PatientHttp::RequestTask] The request task.
|
|
67
76
|
# @return [void]
|
|
68
77
|
def unregister(task)
|
|
69
78
|
task_id = full_task_id(task.id)
|
|
@@ -75,14 +84,15 @@ module PatientHttp
|
|
|
75
84
|
raise if PatientHttp.testing?
|
|
76
85
|
end
|
|
77
86
|
|
|
78
|
-
#
|
|
79
|
-
# it on its next pass.
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
#
|
|
87
|
+
# Releases a request from this process, so the orphan collector
|
|
88
|
+
# re-enqueues it on its next pass.
|
|
89
|
+
#
|
|
90
|
+
# Use this when a result can't be delivered. This process no longer
|
|
91
|
+
# tracks the request, so its record must not look like it belongs to a
|
|
92
|
+
# live process. Orphan collection skips records whose process is still
|
|
93
|
+
# registered, which would strand the request until this process exits.
|
|
84
94
|
#
|
|
85
|
-
# @param task [PatientHttp::RequestTask]
|
|
95
|
+
# @param task [PatientHttp::RequestTask] The request task to release.
|
|
86
96
|
# @return [void]
|
|
87
97
|
def release(task)
|
|
88
98
|
task_id = full_task_id(task.id)
|
|
@@ -97,9 +107,9 @@ module PatientHttp
|
|
|
97
107
|
raise if PatientHttp.testing?
|
|
98
108
|
end
|
|
99
109
|
|
|
100
|
-
#
|
|
110
|
+
# Updates the heartbeat times of requests in one query.
|
|
101
111
|
#
|
|
102
|
-
# @param task_ids [Array<String>]
|
|
112
|
+
# @param task_ids [Array<String>] The request IDs.
|
|
103
113
|
# @return [void]
|
|
104
114
|
def update_heartbeats(task_ids)
|
|
105
115
|
return if task_ids.empty?
|
|
@@ -113,7 +123,7 @@ module PatientHttp
|
|
|
113
123
|
raise if PatientHttp.testing?
|
|
114
124
|
end
|
|
115
125
|
|
|
116
|
-
#
|
|
126
|
+
# Creates or refreshes this process's registration.
|
|
117
127
|
#
|
|
118
128
|
# @return [void]
|
|
119
129
|
def ping_process
|
|
@@ -130,7 +140,7 @@ module PatientHttp
|
|
|
130
140
|
raise if PatientHttp.testing?
|
|
131
141
|
end
|
|
132
142
|
|
|
133
|
-
#
|
|
143
|
+
# Removes this process from the process registrations.
|
|
134
144
|
#
|
|
135
145
|
# @return [void]
|
|
136
146
|
def remove_process
|
|
@@ -142,14 +152,14 @@ module PatientHttp
|
|
|
142
152
|
raise if PatientHttp.testing?
|
|
143
153
|
end
|
|
144
154
|
|
|
145
|
-
#
|
|
155
|
+
# Tries to acquire the distributed garbage collection lock.
|
|
146
156
|
#
|
|
147
|
-
#
|
|
148
|
-
# process can claim the lock
|
|
149
|
-
# Returns false if another process holds a non-expired lock, or if GC was
|
|
150
|
-
# run recently (within heartbeat_interval).
|
|
157
|
+
# A single semaphore row with pessimistic locking makes sure that only
|
|
158
|
+
# one process at a time can claim the lock.
|
|
151
159
|
#
|
|
152
|
-
# @return [Boolean] true if lock acquired
|
|
160
|
+
# @return [Boolean] `true` if the lock was acquired. `false` if another
|
|
161
|
+
# process holds an unexpired lock, or if garbage collection ran within
|
|
162
|
+
# the last `heartbeat_interval` seconds.
|
|
153
163
|
def acquire_gc_lock
|
|
154
164
|
now = Time.current
|
|
155
165
|
expires_at = now + gc_lock_ttl.seconds
|
|
@@ -181,7 +191,8 @@ module PatientHttp
|
|
|
181
191
|
false
|
|
182
192
|
end
|
|
183
193
|
|
|
184
|
-
#
|
|
194
|
+
# Releases the garbage collection lock if this process holds it, and
|
|
195
|
+
# records the time in `last_gc_at`.
|
|
185
196
|
#
|
|
186
197
|
# @return [void]
|
|
187
198
|
def release_gc_lock
|
|
@@ -192,11 +203,12 @@ module PatientHttp
|
|
|
192
203
|
raise if PatientHttp.testing?
|
|
193
204
|
end
|
|
194
205
|
|
|
195
|
-
#
|
|
206
|
+
# Finds orphaned requests and re-enqueues them.
|
|
196
207
|
#
|
|
197
|
-
# @param orphan_threshold_seconds [Numeric]
|
|
198
|
-
#
|
|
199
|
-
# @
|
|
208
|
+
# @param orphan_threshold_seconds [Numeric] The number of seconds without
|
|
209
|
+
# a heartbeat after which a request is considered orphaned.
|
|
210
|
+
# @param logger [Logger] The logger for output.
|
|
211
|
+
# @return [Integer] The number of orphaned requests re-enqueued.
|
|
200
212
|
def cleanup_orphaned_requests(orphan_threshold_seconds, logger)
|
|
201
213
|
threshold = Time.current - orphan_threshold_seconds.seconds
|
|
202
214
|
|
|
@@ -222,18 +234,19 @@ module PatientHttp
|
|
|
222
234
|
reenqueued_count
|
|
223
235
|
end
|
|
224
236
|
|
|
225
|
-
#
|
|
237
|
+
# Returns the registry ID for a request. The registry ID includes this
|
|
238
|
+
# process's ID.
|
|
226
239
|
#
|
|
227
|
-
# @param task_id [String]
|
|
228
|
-
# @return [String]
|
|
240
|
+
# @param task_id [String] The request ID.
|
|
241
|
+
# @return [String] The registry ID.
|
|
229
242
|
def full_task_id(task_id)
|
|
230
243
|
"#{@lock_identifier}/#{task_id}"
|
|
231
244
|
end
|
|
232
245
|
|
|
233
|
-
#
|
|
246
|
+
# Returns whether a request is in the registry.
|
|
234
247
|
#
|
|
235
|
-
# @param task [PatientHttp::RequestTask]
|
|
236
|
-
# @return [Boolean]
|
|
248
|
+
# @param task [PatientHttp::RequestTask] The request task.
|
|
249
|
+
# @return [Boolean] `true` if the request is registered.
|
|
237
250
|
# @api private
|
|
238
251
|
def registered?(task)
|
|
239
252
|
with_connection do
|
|
@@ -241,9 +254,9 @@ module PatientHttp
|
|
|
241
254
|
end
|
|
242
255
|
end
|
|
243
256
|
|
|
244
|
-
#
|
|
257
|
+
# Deletes all records. Works only in test mode.
|
|
245
258
|
#
|
|
246
|
-
# @raise [RuntimeError]
|
|
259
|
+
# @raise [RuntimeError] If called outside test mode.
|
|
247
260
|
# @return [void]
|
|
248
261
|
# @api private
|
|
249
262
|
def self.clear_all!
|
|
@@ -258,9 +271,9 @@ module PatientHttp
|
|
|
258
271
|
|
|
259
272
|
private
|
|
260
273
|
|
|
261
|
-
#
|
|
262
|
-
# gem
|
|
263
|
-
#
|
|
274
|
+
# Checks out a database connection only for the duration of the work, so
|
|
275
|
+
# the gem's threads don't hold pool connections between operations. The
|
|
276
|
+
# gem's threads are the completion workers and the monitor thread.
|
|
264
277
|
def with_connection(&block)
|
|
265
278
|
Record.connection_pool.with_connection(&block)
|
|
266
279
|
end
|
|
@@ -275,29 +288,34 @@ module PatientHttp
|
|
|
275
288
|
GcLock.insert_all([{lock_name: GC_LOCK_NAME}])
|
|
276
289
|
end
|
|
277
290
|
|
|
278
|
-
#
|
|
279
|
-
# resolves conflicts
|
|
280
|
-
#
|
|
291
|
+
# Returns the conflict target for an upsert. MySQL doesn't support an
|
|
292
|
+
# explicit conflict target. It resolves conflicts through the table's
|
|
293
|
+
# unique indexes with `ON DUPLICATE KEY UPDATE`. Adapters that support
|
|
294
|
+
# conflict targets, such as PostgreSQL and SQLite, require one.
|
|
281
295
|
#
|
|
282
|
-
# @param column [Symbol]
|
|
283
|
-
# @return [Symbol, nil]
|
|
296
|
+
# @param column [Symbol] The unique column to use as the conflict target.
|
|
297
|
+
# @return [Symbol, nil] The column, or `nil` if the adapter doesn't allow
|
|
298
|
+
# a target.
|
|
284
299
|
def upsert_unique_by(column)
|
|
285
300
|
Record.connection.supports_insert_conflict_target? ? column : nil
|
|
286
301
|
end
|
|
287
302
|
|
|
288
|
-
# Re-
|
|
303
|
+
# Re-enqueues one orphaned record.
|
|
289
304
|
#
|
|
290
|
-
#
|
|
291
|
-
# heartbeat
|
|
292
|
-
#
|
|
293
|
-
#
|
|
294
|
-
#
|
|
295
|
-
#
|
|
305
|
+
# The method claims the record with an update that matches its exact
|
|
306
|
+
# heartbeat, which handles race conditions. If the heartbeat changed
|
|
307
|
+
# between the read and the claim, the update matches no rows and the
|
|
308
|
+
# method skips the record. The claim refreshes the heartbeat before the
|
|
309
|
+
# job is enqueued. If the process crashes during recovery, the record
|
|
310
|
+
# goes stale again and a later garbage collection pass retries it, so the
|
|
311
|
+
# request isn't lost. The record is deleted only after the job is
|
|
312
|
+
# enqueued.
|
|
296
313
|
#
|
|
297
|
-
# @param record [InflightRequest]
|
|
298
|
-
# @param threshold [
|
|
299
|
-
#
|
|
300
|
-
# @
|
|
314
|
+
# @param record [InflightRequest] The orphaned record.
|
|
315
|
+
# @param threshold [Time] The heartbeat cutoff. Only records with an
|
|
316
|
+
# older heartbeat are orphaned.
|
|
317
|
+
# @param logger [Logger] The logger for output.
|
|
318
|
+
# @return [Boolean] `true` if the record was re-enqueued.
|
|
301
319
|
def reenqueue_orphaned_record(record, threshold, logger)
|
|
302
320
|
# Atomically claim only if still orphaned (heartbeat unchanged). The dead
|
|
303
321
|
# process_id is left in place so a failed recovery becomes orphaned again.
|
|
@@ -2,26 +2,27 @@
|
|
|
2
2
|
|
|
3
3
|
module PatientHttp
|
|
4
4
|
module SolidQueue
|
|
5
|
-
# Background thread that
|
|
6
|
-
#
|
|
5
|
+
# Background thread that updates heartbeats for in-flight requests and
|
|
6
|
+
# re-enqueues orphaned requests.
|
|
7
7
|
class TaskMonitorThread
|
|
8
8
|
include PatientHttp::TimeHelper
|
|
9
9
|
|
|
10
10
|
# Maximum seconds to sleep between monitor thread checks.
|
|
11
11
|
MAX_MONITOR_SLEEP = 5.0
|
|
12
12
|
|
|
13
|
-
# @return [Configuration]
|
|
13
|
+
# @return [Configuration] The gem configuration.
|
|
14
14
|
attr_reader :config
|
|
15
15
|
|
|
16
|
-
# @return [TaskMonitor]
|
|
16
|
+
# @return [TaskMonitor] The in-flight request registry.
|
|
17
17
|
attr_reader :task_monitor
|
|
18
18
|
|
|
19
|
-
#
|
|
19
|
+
# Creates the monitor thread. Call {#start} to run it.
|
|
20
20
|
#
|
|
21
|
-
# @param config [Configuration]
|
|
22
|
-
# @param task_monitor [TaskMonitor]
|
|
23
|
-
# @param tracked_ids_callback [Proc]
|
|
24
|
-
# processors
|
|
21
|
+
# @param config [Configuration] The gem configuration.
|
|
22
|
+
# @param task_monitor [TaskMonitor] The in-flight request registry.
|
|
23
|
+
# @param tracked_ids_callback [Proc] A callable that returns the IDs of
|
|
24
|
+
# all requests that the processors track, including queued, pending,
|
|
25
|
+
# and in-flight requests.
|
|
25
26
|
def initialize(config, task_monitor, tracked_ids_callback)
|
|
26
27
|
@config = config
|
|
27
28
|
@task_monitor = task_monitor
|
|
@@ -31,7 +32,7 @@ module PatientHttp
|
|
|
31
32
|
@stop_signal = Concurrent::Event.new
|
|
32
33
|
end
|
|
33
34
|
|
|
34
|
-
#
|
|
35
|
+
# Starts the thread. Has no effect if the thread is running.
|
|
35
36
|
#
|
|
36
37
|
# @return [void]
|
|
37
38
|
def start
|
|
@@ -50,7 +51,8 @@ module PatientHttp
|
|
|
50
51
|
@thread.name = "async-http-monitor"
|
|
51
52
|
end
|
|
52
53
|
|
|
53
|
-
#
|
|
54
|
+
# Stops the thread. Waits up to 1 second for the thread to finish, and
|
|
55
|
+
# then kills it.
|
|
54
56
|
#
|
|
55
57
|
# @return [void]
|
|
56
58
|
def stop
|
|
@@ -61,9 +63,9 @@ module PatientHttp
|
|
|
61
63
|
@thread = nil
|
|
62
64
|
end
|
|
63
65
|
|
|
64
|
-
#
|
|
66
|
+
# Returns whether the thread is running.
|
|
65
67
|
#
|
|
66
|
-
# @return [Boolean]
|
|
68
|
+
# @return [Boolean] `true` if the thread is running.
|
|
67
69
|
def running?
|
|
68
70
|
@running.true?
|
|
69
71
|
end
|