exekutor 0.1.0 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -3
- data/exe/exekutor +2 -2
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
- data/lib/exekutor/asynchronous.rb +143 -75
- data/lib/exekutor/cleanup.rb +27 -28
- data/lib/exekutor/configuration.rb +102 -48
- data/lib/exekutor/hook.rb +15 -11
- data/lib/exekutor/info/worker.rb +3 -3
- data/lib/exekutor/internal/base_record.rb +2 -1
- data/lib/exekutor/internal/callbacks.rb +55 -35
- data/lib/exekutor/internal/cli/app.rb +33 -23
- data/lib/exekutor/internal/cli/application_loader.rb +17 -6
- data/lib/exekutor/internal/cli/cleanup.rb +54 -40
- data/lib/exekutor/internal/cli/daemon.rb +9 -11
- data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
- data/lib/exekutor/internal/cli/info.rb +117 -84
- data/lib/exekutor/internal/cli/manager.rb +234 -123
- data/lib/exekutor/internal/configuration_builder.rb +49 -30
- data/lib/exekutor/internal/database_connection.rb +6 -0
- data/lib/exekutor/internal/executable.rb +12 -7
- data/lib/exekutor/internal/executor.rb +50 -21
- data/lib/exekutor/internal/hooks.rb +11 -8
- data/lib/exekutor/internal/listener.rb +85 -43
- data/lib/exekutor/internal/logger.rb +29 -10
- data/lib/exekutor/internal/provider.rb +96 -77
- data/lib/exekutor/internal/reserver.rb +66 -19
- data/lib/exekutor/internal/status_server.rb +87 -54
- data/lib/exekutor/job.rb +1 -1
- data/lib/exekutor/job_error.rb +1 -1
- data/lib/exekutor/job_options.rb +22 -13
- data/lib/exekutor/plugins/appsignal.rb +7 -5
- data/lib/exekutor/plugins.rb +8 -4
- data/lib/exekutor/queue.rb +69 -30
- data/lib/exekutor/version.rb +1 -1
- data/lib/exekutor/worker.rb +89 -48
- data/lib/exekutor.rb +2 -2
- data/lib/generators/exekutor/configuration_generator.rb +11 -6
- data/lib/generators/exekutor/install_generator.rb +24 -15
- data/lib/generators/exekutor/templates/install/functions/exekutor_broadcast_job_enqueued.sql +10 -0
- data/lib/generators/exekutor/templates/install/functions/exekutor_requeue_orphaned_jobs.sql +11 -0
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +23 -22
- data/lib/generators/exekutor/templates/install/triggers/exekutor_broadcast_job_enqueued.sql +7 -0
- data/lib/generators/exekutor/templates/install/triggers/exekutor_requeue_orphaned_jobs.sql +5 -0
- data.tar.gz.sig +0 -0
- metadata +67 -23
- metadata.gz.sig +0 -0
- data/lib/generators/exekutor/templates/install/functions/job_notifier.sql +0 -7
- data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql +0 -7
- data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql +0 -6
- data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql +0 -5
@@ -14,46 +14,47 @@ module Exekutor
|
|
14
14
|
# Indicates an unset value
|
15
15
|
# @private
|
16
16
|
DEFAULT_VALUE = Object.new.freeze
|
17
|
-
private_constant
|
17
|
+
private_constant :DEFAULT_VALUE
|
18
18
|
|
19
19
|
# Sets option values in bulk
|
20
20
|
# @return [self]
|
21
21
|
def set(**options)
|
22
22
|
invalid_options = options.keys - __option_names
|
23
23
|
if invalid_options.present?
|
24
|
-
raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{
|
24
|
+
raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{
|
25
|
+
invalid_options.map(&:inspect).join(", ")}"
|
25
26
|
end
|
26
27
|
|
27
28
|
options.each do |name, value|
|
28
|
-
send "#{name}=", value
|
29
|
+
send :"#{name}=", value
|
29
30
|
end
|
30
31
|
self
|
31
32
|
end
|
32
33
|
|
33
|
-
|
34
|
+
class_methods do # rubocop:disable Metrics/BlockLength
|
34
35
|
# Defines a configuration option with the given name.
|
35
36
|
# @param name [Symbol] the name of the option
|
36
|
-
# @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be
|
37
|
-
#
|
37
|
+
# @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be
|
38
|
+
# allowed.
|
39
|
+
# @param type [Class,Array<Class>] the allowed value types. If set the value must be an instance of any of the
|
40
|
+
# given classes.
|
38
41
|
# @param enum [Array<Any>] the allowed values. If set the value must be one of the given values.
|
39
42
|
# @param range [Range] the allowed value range. If set the value must be included in this range.
|
40
43
|
# @param default [Any] the default value
|
41
44
|
# @param reader [Symbol] the name of the reader method
|
45
|
+
# rubocop:disable Metrics/ParameterLists
|
42
46
|
def define_option(name, required: false, type: nil, enum: nil, range: nil, default: DEFAULT_VALUE,
|
43
|
-
reader: name)
|
47
|
+
reader: name, &block)
|
48
|
+
# rubocop:enable Metrics/ParameterLists
|
44
49
|
__option_names << name
|
45
|
-
if reader
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
define_method "#{name}=" do |value|
|
50
|
+
define_reader(reader, name, default) if reader
|
51
|
+
define_writer(name, required, type, enum, range, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def define_writer(name, required, type, enum, range)
|
57
|
+
define_method :"#{name}=" do |value|
|
57
58
|
validate_option_presence! name, value if required
|
58
59
|
validate_option_type! name, value, *type if type.present?
|
59
60
|
validate_option_enum! name, value, *enum if enum.present?
|
@@ -64,14 +65,26 @@ module Exekutor
|
|
64
65
|
self
|
65
66
|
end
|
66
67
|
end
|
68
|
+
|
69
|
+
def define_reader(name, variable_name, default)
|
70
|
+
define_method name do
|
71
|
+
if instance_variable_defined? :"@#{variable_name}"
|
72
|
+
instance_variable_get :"@#{variable_name}"
|
73
|
+
elsif default.respond_to? :call
|
74
|
+
default.call
|
75
|
+
elsif default != DEFAULT_VALUE
|
76
|
+
default
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
67
80
|
end
|
68
81
|
|
69
82
|
# Validates whether the option is present for configuration values that are required
|
70
83
|
# raise [StandardError] if the value is nil or blank
|
71
84
|
def validate_option_presence!(name, value)
|
72
|
-
|
73
|
-
|
74
|
-
|
85
|
+
return if value.present? || value.is_a?(FalseClass)
|
86
|
+
|
87
|
+
raise error_class, "##{name} cannot be #{value.nil? ? "nil" : "blank"}"
|
75
88
|
end
|
76
89
|
|
77
90
|
# Validates whether the value class is allowed
|
@@ -79,7 +92,10 @@ module Exekutor
|
|
79
92
|
def validate_option_type!(name, value, *allowed_types)
|
80
93
|
return if allowed_types.include?(value.class)
|
81
94
|
|
82
|
-
raise error_class, "##{name} should be an instance of #{
|
95
|
+
raise error_class, "##{name} should be an instance of #{
|
96
|
+
allowed_types.map { |c| c == NilClass || c.nil? ? "nil" : c }
|
97
|
+
.to_sentence(two_words_connector: " or ", last_word_connector: ", or ")
|
98
|
+
} (Actual: #{value.class})"
|
83
99
|
end
|
84
100
|
|
85
101
|
# Validates whether the value is a valid enum option
|
@@ -87,7 +103,10 @@ module Exekutor
|
|
87
103
|
def validate_option_enum!(name, value, *allowed_values)
|
88
104
|
return if allowed_values.include?(value)
|
89
105
|
|
90
|
-
raise error_class, "##{name} should be one of #{
|
106
|
+
raise error_class, "##{name} should be one of #{
|
107
|
+
allowed_values.map(&:inspect)
|
108
|
+
.to_sentence(two_words_connector: " or ", last_word_connector: ", or ")
|
109
|
+
}"
|
91
110
|
end
|
92
111
|
|
93
112
|
# Validates whether the value falls in the allowed range
|
@@ -95,10 +114,10 @@ module Exekutor
|
|
95
114
|
def validate_option_range!(name, value, allowed_range)
|
96
115
|
return if allowed_range.include?(value)
|
97
116
|
|
98
|
-
raise error_class,
|
99
|
-
|
100
|
-
|
101
|
-
|
117
|
+
raise error_class, <<~MSG
|
118
|
+
##{name} should be between #{allowed_range.first.inspect} and #{allowed_range.last.inspect}#{
|
119
|
+
" (exclusive)" if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?}
|
120
|
+
MSG
|
102
121
|
end
|
103
122
|
|
104
123
|
protected
|
@@ -106,8 +125,8 @@ module Exekutor
|
|
106
125
|
# The error class to raise when an invalid option value is set
|
107
126
|
# @return [Class<StandardError>]
|
108
127
|
def error_class
|
109
|
-
raise "Implementing class should override #error_class"
|
128
|
+
raise NotImplementedError, "Implementing class should override #error_class"
|
110
129
|
end
|
111
130
|
end
|
112
131
|
end
|
113
|
-
end
|
132
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Exekutor
|
2
4
|
# @private
|
3
5
|
module Internal
|
@@ -9,10 +11,14 @@ module Exekutor
|
|
9
11
|
end
|
10
12
|
|
11
13
|
# The connection name for the specified worker id and process
|
14
|
+
# @param id [String] the id of the worker
|
15
|
+
# @param process [nil,String] the process name
|
12
16
|
def self.application_name(id, process = nil)
|
13
17
|
"Exekutor[id: #{id}]#{" #{process}" if process}"
|
14
18
|
end
|
15
19
|
|
20
|
+
# Reconnects the database if it is not active
|
21
|
+
# @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] the connection adapter to use
|
16
22
|
def self.ensure_active!(connection = BaseRecord.connection)
|
17
23
|
connection.reconnect! unless connection.active?
|
18
24
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Exekutor
|
2
4
|
# Contains internal classes
|
3
5
|
# @private
|
@@ -7,6 +9,7 @@ module Exekutor
|
|
7
9
|
# Possible states
|
8
10
|
STATES = %i[pending started stopped crashed killed].freeze
|
9
11
|
|
12
|
+
# Initializes the internal variables
|
10
13
|
def initialize
|
11
14
|
@state = Concurrent::AtomicReference.new(:pending)
|
12
15
|
@consecutive_errors = Concurrent::AtomicFixnum.new(0)
|
@@ -23,17 +26,18 @@ module Exekutor
|
|
23
26
|
@state.get
|
24
27
|
end
|
25
28
|
|
26
|
-
#
|
29
|
+
# @return [Boolean] whether the state equals +:started+
|
27
30
|
def running?
|
28
31
|
@state.get == :started
|
29
32
|
end
|
30
33
|
|
34
|
+
# @return [Concurrent::AtomicFixnum] the number of consecutive errors that have occurred
|
31
35
|
def consecutive_errors
|
32
36
|
@consecutive_errors
|
33
37
|
end
|
34
38
|
|
35
|
-
# Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first
|
36
|
-
# to 10 minutes from the 13th error on.
|
39
|
+
# Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first
|
40
|
+
# error to 10 minutes from the 13th error on.
|
37
41
|
# @return [Float] The delay
|
38
42
|
def restart_delay
|
39
43
|
if @consecutive_errors.value > 150
|
@@ -41,7 +45,7 @@ module Exekutor
|
|
41
45
|
Exekutor.on_fatal_error error
|
42
46
|
raise error
|
43
47
|
end
|
44
|
-
delay = (9 + @consecutive_errors.value
|
48
|
+
delay = (9 + (@consecutive_errors.value**2.5))
|
45
49
|
delay += delay * (rand(-5..5) / 100.0)
|
46
50
|
delay.clamp(10.0, 600.0)
|
47
51
|
end
|
@@ -50,7 +54,8 @@ module Exekutor
|
|
50
54
|
|
51
55
|
# Changes the state to the given value if the current state matches the expected state. Does nothing otherwise.
|
52
56
|
# @param expected_state [:pending,:started,:stopped,:crashed] the expected state
|
53
|
-
# @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the
|
57
|
+
# @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the
|
58
|
+
# expected
|
54
59
|
# @raise ArgumentError if an invalid state was passed
|
55
60
|
def compare_and_set_state(expected_state, new_state)
|
56
61
|
validate_state! new_state
|
@@ -59,7 +64,7 @@ module Exekutor
|
|
59
64
|
|
60
65
|
# Updates the state to the given value
|
61
66
|
# @raise ArgumentError if an invalid state was passed
|
62
|
-
def
|
67
|
+
def state=(new_state)
|
63
68
|
validate_state! new_state
|
64
69
|
@state.set new_state
|
65
70
|
end
|
@@ -72,4 +77,4 @@ module Exekutor
|
|
72
77
|
end
|
73
78
|
end
|
74
79
|
end
|
75
|
-
end
|
80
|
+
end
|
@@ -15,6 +15,15 @@ module Exekutor
|
|
15
15
|
define_callbacks :after_execute, freeze: true
|
16
16
|
attr_reader :pending_job_updates
|
17
17
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists
|
19
|
+
|
20
|
+
# Create a new executor
|
21
|
+
# @param min_threads [Integer] the minimum number of threads that should be active
|
22
|
+
# @param max_threads [Integer] the maximum number of threads that may be active
|
23
|
+
# @param max_thread_idletime [Integer] the amount of seconds a thread may be idle before being reclaimed
|
24
|
+
# @param delete_completed_jobs [Boolean] whether to delete jobs that complete successfully
|
25
|
+
# @param delete_discarded_jobs [Boolean] whether to delete jobs that are discarded
|
26
|
+
# @param delete_failed_jobs [Boolean] whether to delete jobs that fail
|
18
27
|
def initialize(min_threads: 1, max_threads: default_max_threads, max_thread_idletime: 180,
|
19
28
|
delete_completed_jobs: false, delete_discarded_jobs: false, delete_failed_jobs: false)
|
20
29
|
super()
|
@@ -31,14 +40,16 @@ module Exekutor
|
|
31
40
|
}.freeze
|
32
41
|
end
|
33
42
|
|
43
|
+
# rubocop:enable Metrics/ParameterLists
|
44
|
+
|
34
45
|
# Starts the executor
|
35
46
|
def start
|
36
|
-
|
47
|
+
self.state = :started
|
37
48
|
end
|
38
49
|
|
39
50
|
# Stops the executor
|
40
51
|
def stop
|
41
|
-
|
52
|
+
self.state = :stopped
|
42
53
|
|
43
54
|
@executor.shutdown
|
44
55
|
end
|
@@ -54,7 +65,7 @@ module Exekutor
|
|
54
65
|
# Executes the job on one of the execution threads. Releases the job if there is no thread available to execute
|
55
66
|
# the job.
|
56
67
|
def post(job)
|
57
|
-
@executor.post
|
68
|
+
@executor.post(job) { |*args| execute(*args) }
|
58
69
|
@queued_job_ids.append(job[:id])
|
59
70
|
rescue Concurrent::RejectedExecutionError
|
60
71
|
logger.error "Ran out of threads! Releasing job #{job[:id]}"
|
@@ -108,20 +119,16 @@ module Exekutor
|
|
108
119
|
@active_job_ids.delete(job[:id])
|
109
120
|
end
|
110
121
|
|
111
|
-
def _execute(job, start_time:
|
122
|
+
def _execute(job, start_time: Process.clock_gettime(Process::CLOCK_MONOTONIC))
|
112
123
|
raise Exekutor::DiscardJob, "Maximum queue time expired" if queue_time_expired?(job)
|
113
124
|
|
114
|
-
|
115
|
-
Timeout.timeout Float(timeout), JobExecutionTimeout do
|
116
|
-
ActiveJob::Base.execute(job[:payload])
|
117
|
-
end
|
118
|
-
else
|
125
|
+
with_job_execution_timeout job.dig(:options, "execution_timeout") do
|
119
126
|
ActiveJob::Base.execute(job[:payload])
|
120
127
|
end
|
121
128
|
|
122
|
-
on_job_completed(job, runtime:
|
129
|
+
on_job_completed(job, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
|
123
130
|
rescue StandardError, JobExecutionTimeout => e
|
124
|
-
on_job_failed(job, e, runtime:
|
131
|
+
on_job_failed(job, e, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
|
125
132
|
rescue Exception # rubocop:disable Lint/RescueException
|
126
133
|
# Try to release job when an Exception occurs
|
127
134
|
update_job job, status: "p", worker_id: nil
|
@@ -129,15 +136,17 @@ module Exekutor
|
|
129
136
|
end
|
130
137
|
|
131
138
|
def on_job_completed(job, runtime:)
|
132
|
-
|
139
|
+
next_status = "c"
|
140
|
+
if delete_job? next_status
|
133
141
|
delete_job job
|
134
142
|
else
|
135
|
-
update_job job, status:
|
143
|
+
update_job job, status: next_status, runtime: runtime
|
136
144
|
end
|
137
145
|
end
|
138
146
|
|
139
147
|
def on_job_failed(job, error, runtime:)
|
140
|
-
discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any?
|
148
|
+
discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any? { |c| error.is_a? c }
|
149
|
+
next_status = discarded ? "d" : "f"
|
141
150
|
unless discarded
|
142
151
|
Internal::Hooks.on(:job_failure, job, error)
|
143
152
|
log_error error, "Job failed"
|
@@ -147,14 +156,25 @@ module Exekutor
|
|
147
156
|
# Don't consider this as a failure, try again later.
|
148
157
|
update_job job, status: "p", worker_id: nil
|
149
158
|
|
150
|
-
elsif
|
159
|
+
elsif delete_job? next_status
|
151
160
|
delete_job job
|
152
161
|
|
153
|
-
else
|
154
162
|
# Try to update the job and create a JobError record if update succeeds
|
155
|
-
|
156
|
-
|
157
|
-
|
163
|
+
elsif update_job job, status: next_status, runtime: runtime
|
164
|
+
JobError.create(job_id: job[:id], error: error)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def delete_job?(next_status)
|
169
|
+
case next_status
|
170
|
+
when "c"
|
171
|
+
@options[:delete_completed_jobs]
|
172
|
+
when "d"
|
173
|
+
@options[:delete_discarded_jobs]
|
174
|
+
when "f"
|
175
|
+
@options[:delete_failed_jobs]
|
176
|
+
else
|
177
|
+
false
|
158
178
|
end
|
159
179
|
end
|
160
180
|
|
@@ -199,9 +219,18 @@ module Exekutor
|
|
199
219
|
job[:options]["start_execution_before"].to_f <= Time.now.to_f
|
200
220
|
end
|
201
221
|
|
222
|
+
def with_job_execution_timeout(timeout, &block)
|
223
|
+
if timeout
|
224
|
+
Timeout.timeout Float(timeout), JobExecutionTimeout, &block
|
225
|
+
else
|
226
|
+
yield
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
202
230
|
def lost_db_connection?(error)
|
203
|
-
[ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any?
|
204
|
-
|
231
|
+
[ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any? do |error_class|
|
232
|
+
error.is_a? error_class
|
233
|
+
end && !ActiveRecord::Base.connection.active?
|
205
234
|
end
|
206
235
|
|
207
236
|
# The default maximum number of threads. The value is equal to the size of the DB connection pool minus 1, with
|
@@ -1,5 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# The Exekutor namespace
|
1
4
|
module Exekutor
|
2
5
|
module Internal
|
6
|
+
# The internal implementation of the Exekutor hooks
|
3
7
|
class Hooks
|
4
8
|
include Internal::Callbacks
|
5
9
|
|
@@ -14,14 +18,15 @@ module Exekutor
|
|
14
18
|
def register(callback = nil, &block)
|
15
19
|
if callback
|
16
20
|
callback = callback.new if callback.is_a? Class
|
17
|
-
raise
|
21
|
+
raise "callback must respond to #callbacks" unless callback.respond_to? :callbacks
|
22
|
+
|
18
23
|
callback.callbacks.each do |type, callbacks|
|
19
|
-
callbacks.each { |
|
24
|
+
callbacks.each { |cb| add_callback! type, [], cb }
|
20
25
|
end
|
21
|
-
elsif block.arity
|
22
|
-
block.call self
|
23
|
-
else
|
26
|
+
elsif block.arity.zero?
|
24
27
|
instance_eval(&block)
|
28
|
+
else
|
29
|
+
yield self
|
25
30
|
end
|
26
31
|
end
|
27
32
|
|
@@ -82,6 +87,4 @@ module Exekutor
|
|
82
87
|
# @!attribute [r] hooks
|
83
88
|
# @return [Internal::Hooks] The hooks for exekutor.
|
84
89
|
mattr_reader :hooks, default: Internal::Hooks.new
|
85
|
-
|
86
|
-
# TODO register_hook method instead of `hooks.register`?
|
87
|
-
end
|
90
|
+
end
|
@@ -7,13 +7,16 @@ module Exekutor
|
|
7
7
|
module Internal
|
8
8
|
# Listens for jobs to be executed
|
9
9
|
class Listener
|
10
|
-
include Executable
|
10
|
+
include Executable
|
11
|
+
include Logger
|
11
12
|
|
12
13
|
# The PG notification channel for enqueued jobs
|
13
14
|
JOB_ENQUEUED_CHANNEL = "exekutor::job_enqueued"
|
14
15
|
# The PG notification channel for a worker. Must be formatted with the worker ID.
|
15
16
|
PROVIDER_CHANNEL = "exekutor::worker::%s"
|
16
17
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists
|
19
|
+
|
17
20
|
# Creates a new listener
|
18
21
|
# @param worker_id [String] the ID of the worker
|
19
22
|
# @param queues [Array<String>] the queues to watch
|
@@ -21,11 +24,14 @@ module Exekutor
|
|
21
24
|
# @param pool [ThreadPoolExecutor] the thread pool to use
|
22
25
|
# @param wait_timeout [Integer] the time to listen for notifications
|
23
26
|
# @param set_db_connection_name [Boolean] whether to set the application name on the DB connection
|
24
|
-
def initialize(worker_id:, queues: nil,
|
27
|
+
def initialize(worker_id:, provider:, pool:, queues: nil, min_priority: nil, max_priority: nil, wait_timeout: 60,
|
28
|
+
set_db_connection_name: false)
|
25
29
|
super()
|
26
30
|
@config = {
|
27
31
|
worker_id: worker_id,
|
28
|
-
queues: queues
|
32
|
+
queues: queues.presence,
|
33
|
+
min_priority: min_priority,
|
34
|
+
max_priority: max_priority,
|
29
35
|
wait_timeout: wait_timeout,
|
30
36
|
set_db_connection_name: set_db_connection_name
|
31
37
|
}
|
@@ -37,6 +43,8 @@ module Exekutor
|
|
37
43
|
@listening = Concurrent::AtomicBoolean.new false
|
38
44
|
end
|
39
45
|
|
46
|
+
# rubocop:enable Metrics/ParameterLists
|
47
|
+
|
40
48
|
# Starts the listener
|
41
49
|
def start
|
42
50
|
return false unless compare_and_set_state :pending, :started
|
@@ -47,11 +55,11 @@ module Exekutor
|
|
47
55
|
|
48
56
|
# Stops the listener
|
49
57
|
def stop
|
50
|
-
|
58
|
+
self.state = :stopped
|
51
59
|
begin
|
52
60
|
Exekutor::Job.connection.execute(%(NOTIFY "#{provider_channel}"))
|
53
61
|
rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
|
54
|
-
#ignored
|
62
|
+
# ignored
|
55
63
|
end
|
56
64
|
end
|
57
65
|
|
@@ -62,16 +70,33 @@ module Exekutor
|
|
62
70
|
PROVIDER_CHANNEL % @config[:worker_id]
|
63
71
|
end
|
64
72
|
|
65
|
-
#
|
66
|
-
|
73
|
+
# @return [Boolean] Whether the job matches the configured queues and priority range
|
74
|
+
def job_filter_match?(job_info)
|
75
|
+
listening_to_queue?(job_info["q"]) && listening_to_priority?(job_info["p"].to_i)
|
76
|
+
end
|
77
|
+
|
78
|
+
# @return [Boolean] Whether this listener is listening to the given queue
|
67
79
|
def listening_to_queue?(queue)
|
68
80
|
queues = @config[:queues]
|
69
|
-
queues.
|
81
|
+
queues.nil? || queues.include?(queue)
|
82
|
+
end
|
83
|
+
|
84
|
+
# @return [Boolean] Whether this listener is listening to the given priority
|
85
|
+
def listening_to_priority?(priority)
|
86
|
+
minimum = @config[:min_priority]
|
87
|
+
maximum = @config[:max_priority]
|
88
|
+
(minimum.nil? || minimum <= priority) && (maximum.nil? || maximum >= priority)
|
70
89
|
end
|
71
90
|
|
72
91
|
# Starts the listener thread
|
73
|
-
def start_thread
|
74
|
-
|
92
|
+
def start_thread(delay: nil)
|
93
|
+
return unless running?
|
94
|
+
|
95
|
+
if delay
|
96
|
+
Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
|
97
|
+
else
|
98
|
+
@pool.post { run }
|
99
|
+
end
|
75
100
|
end
|
76
101
|
|
77
102
|
# Sets up the PG notifications and listens for new jobs
|
@@ -79,30 +104,33 @@ module Exekutor
|
|
79
104
|
return unless running? && @thread_running.make_true
|
80
105
|
|
81
106
|
with_pg_connection do |connection|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
connection.exec("UNLISTEN *")
|
89
|
-
end
|
90
|
-
end
|
91
|
-
rescue StandardError => err
|
92
|
-
Exekutor.on_fatal_error err, "[Listener] Runtime error!"
|
93
|
-
set_state :crashed if err.is_a? UnsupportedDatabase
|
94
|
-
|
95
|
-
if running?
|
96
|
-
consecutive_errors.increment
|
97
|
-
delay = restart_delay
|
98
|
-
logger.info "Restarting in %0.1f seconds…" % [delay]
|
99
|
-
Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
|
107
|
+
connection.exec(%(LISTEN "#{provider_channel}"))
|
108
|
+
connection.exec(%(LISTEN "#{JOB_ENQUEUED_CHANNEL}"))
|
109
|
+
consecutive_errors.value = 0
|
110
|
+
catch(:shutdown) { wait_for_jobs(connection) }
|
111
|
+
ensure
|
112
|
+
connection.exec("UNLISTEN *")
|
100
113
|
end
|
114
|
+
rescue StandardError => e
|
115
|
+
on_thread_error(e)
|
101
116
|
ensure
|
102
117
|
@thread_running.make_false
|
103
118
|
@listening.make_false
|
104
119
|
end
|
105
120
|
|
121
|
+
# Called when an error is raised in #run
|
122
|
+
def on_thread_error(error)
|
123
|
+
Exekutor.on_fatal_error error, "[Listener] Runtime error!"
|
124
|
+
self.state = :crashed if error.is_a? UnsupportedDatabase
|
125
|
+
|
126
|
+
return unless running?
|
127
|
+
|
128
|
+
consecutive_errors.increment
|
129
|
+
delay = restart_delay
|
130
|
+
logger.info format("Restarting in %0.1f seconds…", delay)
|
131
|
+
start_thread delay: delay
|
132
|
+
end
|
133
|
+
|
106
134
|
# Listens for jobs. Blocks until the listener is stopped
|
107
135
|
def wait_for_jobs(connection)
|
108
136
|
while running?
|
@@ -112,25 +140,20 @@ module Exekutor
|
|
112
140
|
next unless channel == JOB_ENQUEUED_CHANNEL
|
113
141
|
|
114
142
|
job_info = begin
|
115
|
-
|
116
|
-
rescue
|
117
|
-
logger.error
|
118
|
-
|
143
|
+
JobParser.parse(payload)
|
144
|
+
rescue StandardError => e
|
145
|
+
logger.error e.message
|
146
|
+
nil
|
119
147
|
end
|
120
|
-
unless
|
121
|
-
|
122
|
-
|
123
|
-
end
|
124
|
-
next unless listening_to_queue? job_info["q"]
|
125
|
-
|
126
|
-
scheduled_at = job_info["t"].to_f
|
127
|
-
@provider.update_earliest_scheduled_at(scheduled_at)
|
148
|
+
next unless job_info && job_filter_match?(job_info)
|
149
|
+
|
150
|
+
@provider.update_earliest_scheduled_at(job_info["t"].to_f)
|
128
151
|
end
|
129
152
|
end
|
130
153
|
end
|
131
154
|
|
132
|
-
# Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is
|
133
|
-
# Closes the connection after yielding it to the given block.
|
155
|
+
# Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is
|
156
|
+
# true. Closes the connection after yielding it to the given block.
|
134
157
|
# (Grabbed from PG adapter for action cable)
|
135
158
|
# @yield yields the connection
|
136
159
|
# @yieldparam connection [PG::Connection] the DB connection
|
@@ -159,6 +182,7 @@ module Exekutor
|
|
159
182
|
raise UnsupportedDatabase,
|
160
183
|
"The raw connection of the active record connection adapter must be an instance of PG::Connection"
|
161
184
|
end
|
185
|
+
true
|
162
186
|
end
|
163
187
|
|
164
188
|
# For testing purposes
|
@@ -166,10 +190,28 @@ module Exekutor
|
|
166
190
|
@listening.true?
|
167
191
|
end
|
168
192
|
|
193
|
+
# Parses a NOTIFY payload to a job
|
194
|
+
class JobParser
|
195
|
+
JOB_INFO_KEYS = %w[id q p t].freeze
|
196
|
+
|
197
|
+
def self.parse(payload)
|
198
|
+
job_info = begin
|
199
|
+
payload.split(";").to_h { |el| el.split(":") }
|
200
|
+
rescue StandardError
|
201
|
+
raise Error, "Invalid notification payload: #{payload}"
|
202
|
+
end
|
203
|
+
if (missing_keys = JOB_INFO_KEYS.select { |n| job_info[n].blank? }).present?
|
204
|
+
raise Error, "[Listener] Notification payload is missing #{missing_keys.join(", ")}"
|
205
|
+
end
|
206
|
+
|
207
|
+
job_info
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
169
211
|
# Raised when an error occurs in the listener.
|
170
212
|
class Error < Exekutor::Error; end
|
171
213
|
|
172
|
-
# Raised when the database connection is not an instance of PG::Connection
|
214
|
+
# Raised when the database connection is not an instance of +PG::Connection+.
|
173
215
|
class UnsupportedDatabase < Exekutor::Error; end
|
174
216
|
end
|
175
217
|
end
|