exekutor 0.1.0 → 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/exe/exekutor +2 -2
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
- data/lib/exekutor/asynchronous.rb +143 -75
- data/lib/exekutor/cleanup.rb +27 -28
- data/lib/exekutor/configuration.rb +48 -25
- data/lib/exekutor/hook.rb +15 -11
- data/lib/exekutor/info/worker.rb +3 -3
- data/lib/exekutor/internal/base_record.rb +2 -1
- data/lib/exekutor/internal/callbacks.rb +55 -35
- data/lib/exekutor/internal/cli/app.rb +31 -23
- data/lib/exekutor/internal/cli/application_loader.rb +17 -6
- data/lib/exekutor/internal/cli/cleanup.rb +54 -40
- data/lib/exekutor/internal/cli/daemon.rb +9 -11
- data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
- data/lib/exekutor/internal/cli/info.rb +117 -84
- data/lib/exekutor/internal/cli/manager.rb +190 -123
- data/lib/exekutor/internal/configuration_builder.rb +40 -27
- data/lib/exekutor/internal/database_connection.rb +6 -0
- data/lib/exekutor/internal/executable.rb +12 -7
- data/lib/exekutor/internal/executor.rb +50 -21
- data/lib/exekutor/internal/hooks.rb +11 -8
- data/lib/exekutor/internal/listener.rb +66 -39
- data/lib/exekutor/internal/logger.rb +28 -10
- data/lib/exekutor/internal/provider.rb +93 -74
- data/lib/exekutor/internal/reserver.rb +27 -12
- data/lib/exekutor/internal/status_server.rb +81 -49
- data/lib/exekutor/job.rb +1 -1
- data/lib/exekutor/job_error.rb +1 -1
- data/lib/exekutor/job_options.rb +22 -13
- data/lib/exekutor/plugins/appsignal.rb +7 -5
- data/lib/exekutor/plugins.rb +8 -4
- data/lib/exekutor/queue.rb +40 -22
- data/lib/exekutor/version.rb +1 -1
- data/lib/exekutor/worker.rb +88 -47
- data/lib/exekutor.rb +2 -2
- data/lib/generators/exekutor/configuration_generator.rb +9 -5
- data/lib/generators/exekutor/install_generator.rb +26 -15
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +11 -10
- data.tar.gz.sig +0 -0
- metadata +63 -19
- metadata.gz.sig +0 -0
@@ -14,14 +14,15 @@ module Exekutor
|
|
14
14
|
# Indicates an unset value
|
15
15
|
# @private
|
16
16
|
DEFAULT_VALUE = Object.new.freeze
|
17
|
-
private_constant
|
17
|
+
private_constant :DEFAULT_VALUE
|
18
18
|
|
19
19
|
# Sets option values in bulk
|
20
20
|
# @return [self]
|
21
21
|
def set(**options)
|
22
22
|
invalid_options = options.keys - __option_names
|
23
23
|
if invalid_options.present?
|
24
|
-
raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{
|
24
|
+
raise error_class, "Invalid option#{"s" if invalid_options.many?}: #{
|
25
|
+
invalid_options.map(&:inspect).join(", ")}"
|
25
26
|
end
|
26
27
|
|
27
28
|
options.each do |name, value|
|
@@ -30,29 +31,29 @@ module Exekutor
|
|
30
31
|
self
|
31
32
|
end
|
32
33
|
|
33
|
-
|
34
|
+
class_methods do # rubocop:disable Metrics/BlockLength
|
34
35
|
# Defines a configuration option with the given name.
|
35
36
|
# @param name [Symbol] the name of the option
|
36
|
-
# @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be
|
37
|
-
#
|
37
|
+
# @param required [Boolean] whether a value is required. If +true+, any +nil+ or +blank?+ value will not be
|
38
|
+
# allowed.
|
39
|
+
# @param type [Class,Array<Class>] the allowed value types. If set the value must be an instance of any of the
|
40
|
+
# given classes.
|
38
41
|
# @param enum [Array<Any>] the allowed values. If set the value must be one of the given values.
|
39
42
|
# @param range [Range] the allowed value range. If set the value must be included in this range.
|
40
43
|
# @param default [Any] the default value
|
41
44
|
# @param reader [Symbol] the name of the reader method
|
45
|
+
# rubocop:disable Metrics/ParameterLists
|
42
46
|
def define_option(name, required: false, type: nil, enum: nil, range: nil, default: DEFAULT_VALUE,
|
43
|
-
reader: name)
|
47
|
+
reader: name, &block)
|
48
|
+
# rubocop:enable Metrics/ParameterLists
|
44
49
|
__option_names << name
|
45
|
-
if reader
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
default
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
50
|
+
define_reader(reader, name, default) if reader
|
51
|
+
define_writer(name, required, type, enum, range, &block)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def define_writer(name, required, type, enum, range)
|
56
57
|
define_method "#{name}=" do |value|
|
57
58
|
validate_option_presence! name, value if required
|
58
59
|
validate_option_type! name, value, *type if type.present?
|
@@ -64,14 +65,26 @@ module Exekutor
|
|
64
65
|
self
|
65
66
|
end
|
66
67
|
end
|
68
|
+
|
69
|
+
def define_reader(name, variable_name, default)
|
70
|
+
define_method name do
|
71
|
+
if instance_variable_defined? :"@#{variable_name}"
|
72
|
+
instance_variable_get :"@#{variable_name}"
|
73
|
+
elsif default.respond_to? :call
|
74
|
+
default.call
|
75
|
+
elsif default != DEFAULT_VALUE
|
76
|
+
default
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
67
80
|
end
|
68
81
|
|
69
82
|
# Validates whether the option is present for configuration values that are required
|
70
83
|
# raise [StandardError] if the value is nil or blank
|
71
84
|
def validate_option_presence!(name, value)
|
72
|
-
|
73
|
-
|
74
|
-
|
85
|
+
return if value.present? || value.is_a?(FalseClass)
|
86
|
+
|
87
|
+
raise error_class, "##{name} cannot be #{value.nil? ? "nil" : "blank"}"
|
75
88
|
end
|
76
89
|
|
77
90
|
# Validates whether the value class is allowed
|
@@ -79,7 +92,8 @@ module Exekutor
|
|
79
92
|
def validate_option_type!(name, value, *allowed_types)
|
80
93
|
return if allowed_types.include?(value.class)
|
81
94
|
|
82
|
-
raise error_class, "##{name} should be an instance of #{
|
95
|
+
raise error_class, "##{name} should be an instance of #{
|
96
|
+
allowed_types.to_sentence(last_word_connector: ", or ")} (Actual: #{value.class})"
|
83
97
|
end
|
84
98
|
|
85
99
|
# Validates whether the value is a valid enum option
|
@@ -87,7 +101,8 @@ module Exekutor
|
|
87
101
|
def validate_option_enum!(name, value, *allowed_values)
|
88
102
|
return if allowed_values.include?(value)
|
89
103
|
|
90
|
-
raise error_class, "##{name} should be one of #{allowed_values.map(&:inspect)
|
104
|
+
raise error_class, "##{name} should be one of #{allowed_values.map(&:inspect)
|
105
|
+
.to_sentence(last_word_connector: ", or ")}"
|
91
106
|
end
|
92
107
|
|
93
108
|
# Validates whether the value falls in the allowed range
|
@@ -96,9 +111,7 @@ module Exekutor
|
|
96
111
|
return if allowed_range.include?(value)
|
97
112
|
|
98
113
|
raise error_class, "##{name} should be between #{allowed_range.first} and #{allowed_range.last}#{
|
99
|
-
if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?
|
100
|
-
" (exclusive)"
|
101
|
-
end}"
|
114
|
+
" (exclusive)" if allowed_range.respond_to?(:exclude_end?) && allowed_range.exclude_end?}"
|
102
115
|
end
|
103
116
|
|
104
117
|
protected
|
@@ -106,8 +119,8 @@ module Exekutor
|
|
106
119
|
# The error class to raise when an invalid option value is set
|
107
120
|
# @return [Class<StandardError>]
|
108
121
|
def error_class
|
109
|
-
raise "Implementing class should override #error_class"
|
122
|
+
raise NotImplementedError, "Implementing class should override #error_class"
|
110
123
|
end
|
111
124
|
end
|
112
125
|
end
|
113
|
-
end
|
126
|
+
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Exekutor
|
2
4
|
# @private
|
3
5
|
module Internal
|
@@ -9,10 +11,14 @@ module Exekutor
|
|
9
11
|
end
|
10
12
|
|
11
13
|
# The connection name for the specified worker id and process
|
14
|
+
# @param id [String] the id of the worker
|
15
|
+
# @param process [nil,String] the process name
|
12
16
|
def self.application_name(id, process = nil)
|
13
17
|
"Exekutor[id: #{id}]#{" #{process}" if process}"
|
14
18
|
end
|
15
19
|
|
20
|
+
# Reconnects the database if it is not active
|
21
|
+
# @param connection [ActiveRecord::ConnectionAdapters::AbstractAdapter] the connection adapter to use
|
16
22
|
def self.ensure_active!(connection = BaseRecord.connection)
|
17
23
|
connection.reconnect! unless connection.active?
|
18
24
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Exekutor
|
2
4
|
# Contains internal classes
|
3
5
|
# @private
|
@@ -7,6 +9,7 @@ module Exekutor
|
|
7
9
|
# Possible states
|
8
10
|
STATES = %i[pending started stopped crashed killed].freeze
|
9
11
|
|
12
|
+
# Initializes the internal variables
|
10
13
|
def initialize
|
11
14
|
@state = Concurrent::AtomicReference.new(:pending)
|
12
15
|
@consecutive_errors = Concurrent::AtomicFixnum.new(0)
|
@@ -23,17 +26,18 @@ module Exekutor
|
|
23
26
|
@state.get
|
24
27
|
end
|
25
28
|
|
26
|
-
#
|
29
|
+
# @return [Boolean] whether the state equals +:started+
|
27
30
|
def running?
|
28
31
|
@state.get == :started
|
29
32
|
end
|
30
33
|
|
34
|
+
# @return [Concurrent::AtomicFixnum] the number of consecutive errors that have occurred
|
31
35
|
def consecutive_errors
|
32
36
|
@consecutive_errors
|
33
37
|
end
|
34
38
|
|
35
|
-
# Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first
|
36
|
-
# to 10 minutes from the 13th error on.
|
39
|
+
# Calculates an exponential delay based on {#consecutive_errors}. The delay ranges from 10 seconds on the first
|
40
|
+
# error to 10 minutes from the 13th error on.
|
37
41
|
# @return [Float] The delay
|
38
42
|
def restart_delay
|
39
43
|
if @consecutive_errors.value > 150
|
@@ -41,7 +45,7 @@ module Exekutor
|
|
41
45
|
Exekutor.on_fatal_error error
|
42
46
|
raise error
|
43
47
|
end
|
44
|
-
delay = (9 + @consecutive_errors.value
|
48
|
+
delay = (9 + (@consecutive_errors.value**2.5))
|
45
49
|
delay += delay * (rand(-5..5) / 100.0)
|
46
50
|
delay.clamp(10.0, 600.0)
|
47
51
|
end
|
@@ -50,7 +54,8 @@ module Exekutor
|
|
50
54
|
|
51
55
|
# Changes the state to the given value if the current state matches the expected state. Does nothing otherwise.
|
52
56
|
# @param expected_state [:pending,:started,:stopped,:crashed] the expected state
|
53
|
-
# @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the
|
57
|
+
# @param new_state [:pending,:started,:stopped,:crashed] the state to change to if the current state matches the
|
58
|
+
# expected
|
54
59
|
# @raise ArgumentError if an invalid state was passed
|
55
60
|
def compare_and_set_state(expected_state, new_state)
|
56
61
|
validate_state! new_state
|
@@ -59,7 +64,7 @@ module Exekutor
|
|
59
64
|
|
60
65
|
# Updates the state to the given value
|
61
66
|
# @raise ArgumentError if an invalid state was passed
|
62
|
-
def
|
67
|
+
def state=(new_state)
|
63
68
|
validate_state! new_state
|
64
69
|
@state.set new_state
|
65
70
|
end
|
@@ -72,4 +77,4 @@ module Exekutor
|
|
72
77
|
end
|
73
78
|
end
|
74
79
|
end
|
75
|
-
end
|
80
|
+
end
|
@@ -15,6 +15,15 @@ module Exekutor
|
|
15
15
|
define_callbacks :after_execute, freeze: true
|
16
16
|
attr_reader :pending_job_updates
|
17
17
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists
|
19
|
+
|
20
|
+
# Create a new executor
|
21
|
+
# @param min_threads [Integer] the minimum number of threads that should be active
|
22
|
+
# @param max_threads [Integer] the maximum number of threads that may be active
|
23
|
+
# @param max_thread_idletime [Integer] the amount of seconds a thread may be idle before being reclaimed
|
24
|
+
# @param delete_completed_jobs [Boolean] whether to delete jobs that complete successfully
|
25
|
+
# @param delete_discarded_jobs [Boolean] whether to delete jobs that are discarded
|
26
|
+
# @param delete_failed_jobs [Boolean] whether to delete jobs that fail
|
18
27
|
def initialize(min_threads: 1, max_threads: default_max_threads, max_thread_idletime: 180,
|
19
28
|
delete_completed_jobs: false, delete_discarded_jobs: false, delete_failed_jobs: false)
|
20
29
|
super()
|
@@ -31,14 +40,16 @@ module Exekutor
|
|
31
40
|
}.freeze
|
32
41
|
end
|
33
42
|
|
43
|
+
# rubocop:enable Metrics/ParameterLists
|
44
|
+
|
34
45
|
# Starts the executor
|
35
46
|
def start
|
36
|
-
|
47
|
+
self.state = :started
|
37
48
|
end
|
38
49
|
|
39
50
|
# Stops the executor
|
40
51
|
def stop
|
41
|
-
|
52
|
+
self.state = :stopped
|
42
53
|
|
43
54
|
@executor.shutdown
|
44
55
|
end
|
@@ -54,7 +65,7 @@ module Exekutor
|
|
54
65
|
# Executes the job on one of the execution threads. Releases the job if there is no thread available to execute
|
55
66
|
# the job.
|
56
67
|
def post(job)
|
57
|
-
@executor.post
|
68
|
+
@executor.post(job) { |*args| execute(*args) }
|
58
69
|
@queued_job_ids.append(job[:id])
|
59
70
|
rescue Concurrent::RejectedExecutionError
|
60
71
|
logger.error "Ran out of threads! Releasing job #{job[:id]}"
|
@@ -108,20 +119,16 @@ module Exekutor
|
|
108
119
|
@active_job_ids.delete(job[:id])
|
109
120
|
end
|
110
121
|
|
111
|
-
def _execute(job, start_time:
|
122
|
+
def _execute(job, start_time: Process.clock_gettime(Process::CLOCK_MONOTONIC))
|
112
123
|
raise Exekutor::DiscardJob, "Maximum queue time expired" if queue_time_expired?(job)
|
113
124
|
|
114
|
-
|
115
|
-
Timeout.timeout Float(timeout), JobExecutionTimeout do
|
116
|
-
ActiveJob::Base.execute(job[:payload])
|
117
|
-
end
|
118
|
-
else
|
125
|
+
with_job_execution_timeout job.dig(:options, "execution_timeout") do
|
119
126
|
ActiveJob::Base.execute(job[:payload])
|
120
127
|
end
|
121
128
|
|
122
|
-
on_job_completed(job, runtime:
|
129
|
+
on_job_completed(job, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
|
123
130
|
rescue StandardError, JobExecutionTimeout => e
|
124
|
-
on_job_failed(job, e, runtime:
|
131
|
+
on_job_failed(job, e, runtime: Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time)
|
125
132
|
rescue Exception # rubocop:disable Lint/RescueException
|
126
133
|
# Try to release job when an Exception occurs
|
127
134
|
update_job job, status: "p", worker_id: nil
|
@@ -129,15 +136,17 @@ module Exekutor
|
|
129
136
|
end
|
130
137
|
|
131
138
|
def on_job_completed(job, runtime:)
|
132
|
-
|
139
|
+
next_status = "c"
|
140
|
+
if delete_job? next_status
|
133
141
|
delete_job job
|
134
142
|
else
|
135
|
-
update_job job, status:
|
143
|
+
update_job job, status: next_status, runtime: runtime
|
136
144
|
end
|
137
145
|
end
|
138
146
|
|
139
147
|
def on_job_failed(job, error, runtime:)
|
140
|
-
discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any?
|
148
|
+
discarded = [Exekutor::DiscardJob, JobExecutionTimeout].any? { |c| error.is_a? c }
|
149
|
+
next_status = discarded ? "d" : "f"
|
141
150
|
unless discarded
|
142
151
|
Internal::Hooks.on(:job_failure, job, error)
|
143
152
|
log_error error, "Job failed"
|
@@ -147,14 +156,25 @@ module Exekutor
|
|
147
156
|
# Don't consider this as a failure, try again later.
|
148
157
|
update_job job, status: "p", worker_id: nil
|
149
158
|
|
150
|
-
elsif
|
159
|
+
elsif delete_job? next_status
|
151
160
|
delete_job job
|
152
161
|
|
153
|
-
else
|
154
162
|
# Try to update the job and create a JobError record if update succeeds
|
155
|
-
|
156
|
-
|
157
|
-
|
163
|
+
elsif update_job job, status: next_status, runtime: runtime
|
164
|
+
JobError.create(job_id: job[:id], error: error)
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def delete_job?(next_status)
|
169
|
+
case next_status
|
170
|
+
when "c"
|
171
|
+
@options[:delete_completed_jobs]
|
172
|
+
when "d"
|
173
|
+
@options[:delete_discarded_jobs]
|
174
|
+
when "f"
|
175
|
+
@options[:delete_failed_jobs]
|
176
|
+
else
|
177
|
+
false
|
158
178
|
end
|
159
179
|
end
|
160
180
|
|
@@ -199,9 +219,18 @@ module Exekutor
|
|
199
219
|
job[:options]["start_execution_before"].to_f <= Time.now.to_f
|
200
220
|
end
|
201
221
|
|
222
|
+
def with_job_execution_timeout(timeout, &block)
|
223
|
+
if timeout
|
224
|
+
Timeout.timeout Float(timeout), JobExecutionTimeout, &block
|
225
|
+
else
|
226
|
+
yield
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
202
230
|
def lost_db_connection?(error)
|
203
|
-
[ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any?
|
204
|
-
|
231
|
+
[ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished].any? do |error_class|
|
232
|
+
error.is_a? error_class
|
233
|
+
end && !ActiveRecord::Base.connection.active?
|
205
234
|
end
|
206
235
|
|
207
236
|
# The default maximum number of threads. The value is equal to the size of the DB connection pool minus 1, with
|
@@ -1,5 +1,9 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# The Exekutor namespace
|
1
4
|
module Exekutor
|
2
5
|
module Internal
|
6
|
+
# The internal implementation of the Exekutor hooks
|
3
7
|
class Hooks
|
4
8
|
include Internal::Callbacks
|
5
9
|
|
@@ -14,14 +18,15 @@ module Exekutor
|
|
14
18
|
def register(callback = nil, &block)
|
15
19
|
if callback
|
16
20
|
callback = callback.new if callback.is_a? Class
|
17
|
-
raise
|
21
|
+
raise "callback must respond to #callbacks" unless callback.respond_to? :callbacks
|
22
|
+
|
18
23
|
callback.callbacks.each do |type, callbacks|
|
19
|
-
callbacks.each { |
|
24
|
+
callbacks.each { |cb| add_callback! type, [], cb }
|
20
25
|
end
|
21
|
-
elsif block.arity
|
22
|
-
block.call self
|
23
|
-
else
|
26
|
+
elsif block.arity.zero?
|
24
27
|
instance_eval(&block)
|
28
|
+
else
|
29
|
+
yield self
|
25
30
|
end
|
26
31
|
end
|
27
32
|
|
@@ -82,6 +87,4 @@ module Exekutor
|
|
82
87
|
# @!attribute [r] hooks
|
83
88
|
# @return [Internal::Hooks] The hooks for exekutor.
|
84
89
|
mattr_reader :hooks, default: Internal::Hooks.new
|
85
|
-
|
86
|
-
# TODO register_hook method instead of `hooks.register`?
|
87
|
-
end
|
90
|
+
end
|
@@ -7,13 +7,16 @@ module Exekutor
|
|
7
7
|
module Internal
|
8
8
|
# Listens for jobs to be executed
|
9
9
|
class Listener
|
10
|
-
include Executable
|
10
|
+
include Executable
|
11
|
+
include Logger
|
11
12
|
|
12
13
|
# The PG notification channel for enqueued jobs
|
13
14
|
JOB_ENQUEUED_CHANNEL = "exekutor::job_enqueued"
|
14
15
|
# The PG notification channel for a worker. Must be formatted with the worker ID.
|
15
16
|
PROVIDER_CHANNEL = "exekutor::worker::%s"
|
16
17
|
|
18
|
+
# rubocop:disable Metrics/ParameterLists
|
19
|
+
|
17
20
|
# Creates a new listener
|
18
21
|
# @param worker_id [String] the ID of the worker
|
19
22
|
# @param queues [Array<String>] the queues to watch
|
@@ -21,7 +24,7 @@ module Exekutor
|
|
21
24
|
# @param pool [ThreadPoolExecutor] the thread pool to use
|
22
25
|
# @param wait_timeout [Integer] the time to listen for notifications
|
23
26
|
# @param set_db_connection_name [Boolean] whether to set the application name on the DB connection
|
24
|
-
def initialize(worker_id:, queues: nil,
|
27
|
+
def initialize(worker_id:, provider:, pool:, queues: nil, wait_timeout: 60, set_db_connection_name: false)
|
25
28
|
super()
|
26
29
|
@config = {
|
27
30
|
worker_id: worker_id,
|
@@ -37,6 +40,8 @@ module Exekutor
|
|
37
40
|
@listening = Concurrent::AtomicBoolean.new false
|
38
41
|
end
|
39
42
|
|
43
|
+
# rubocop:enable Metrics/ParameterLists
|
44
|
+
|
40
45
|
# Starts the listener
|
41
46
|
def start
|
42
47
|
return false unless compare_and_set_state :pending, :started
|
@@ -47,11 +52,11 @@ module Exekutor
|
|
47
52
|
|
48
53
|
# Stops the listener
|
49
54
|
def stop
|
50
|
-
|
55
|
+
self.state = :stopped
|
51
56
|
begin
|
52
57
|
Exekutor::Job.connection.execute(%(NOTIFY "#{provider_channel}"))
|
53
58
|
rescue ActiveRecord::StatementInvalid, ActiveRecord::ConnectionNotEstablished
|
54
|
-
#ignored
|
59
|
+
# ignored
|
55
60
|
end
|
56
61
|
end
|
57
62
|
|
@@ -70,8 +75,14 @@ module Exekutor
|
|
70
75
|
end
|
71
76
|
|
72
77
|
# Starts the listener thread
|
73
|
-
def start_thread
|
74
|
-
|
78
|
+
def start_thread(delay: nil)
|
79
|
+
return unless running?
|
80
|
+
|
81
|
+
if delay
|
82
|
+
Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
|
83
|
+
else
|
84
|
+
@pool.post { run }
|
85
|
+
end
|
75
86
|
end
|
76
87
|
|
77
88
|
# Sets up the PG notifications and listens for new jobs
|
@@ -79,30 +90,33 @@ module Exekutor
|
|
79
90
|
return unless running? && @thread_running.make_true
|
80
91
|
|
81
92
|
with_pg_connection do |connection|
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
connection.exec("UNLISTEN *")
|
89
|
-
end
|
90
|
-
end
|
91
|
-
rescue StandardError => err
|
92
|
-
Exekutor.on_fatal_error err, "[Listener] Runtime error!"
|
93
|
-
set_state :crashed if err.is_a? UnsupportedDatabase
|
94
|
-
|
95
|
-
if running?
|
96
|
-
consecutive_errors.increment
|
97
|
-
delay = restart_delay
|
98
|
-
logger.info "Restarting in %0.1f seconds…" % [delay]
|
99
|
-
Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
|
93
|
+
connection.exec(%(LISTEN "#{provider_channel}"))
|
94
|
+
connection.exec(%(LISTEN "#{JOB_ENQUEUED_CHANNEL}"))
|
95
|
+
consecutive_errors.value = 0
|
96
|
+
catch(:shutdown) { wait_for_jobs(connection) }
|
97
|
+
ensure
|
98
|
+
connection.exec("UNLISTEN *")
|
100
99
|
end
|
100
|
+
rescue StandardError => e
|
101
|
+
on_thread_error(e)
|
101
102
|
ensure
|
102
103
|
@thread_running.make_false
|
103
104
|
@listening.make_false
|
104
105
|
end
|
105
106
|
|
107
|
+
# Called when an error is raised in #run
|
108
|
+
def on_thread_error(error)
|
109
|
+
Exekutor.on_fatal_error error, "[Listener] Runtime error!"
|
110
|
+
self.state = :crashed if error.is_a? UnsupportedDatabase
|
111
|
+
|
112
|
+
return unless running?
|
113
|
+
|
114
|
+
consecutive_errors.increment
|
115
|
+
delay = restart_delay
|
116
|
+
logger.info format("Restarting in %0.1f seconds…", delay)
|
117
|
+
start_thread delay: delay
|
118
|
+
end
|
119
|
+
|
106
120
|
# Listens for jobs. Blocks until the listener is stopped
|
107
121
|
def wait_for_jobs(connection)
|
108
122
|
while running?
|
@@ -112,25 +126,19 @@ module Exekutor
|
|
112
126
|
next unless channel == JOB_ENQUEUED_CHANNEL
|
113
127
|
|
114
128
|
job_info = begin
|
115
|
-
|
116
|
-
rescue
|
117
|
-
logger.error
|
118
|
-
next
|
129
|
+
JobParser.parse(payload)
|
130
|
+
rescue StandardError => e
|
131
|
+
logger.error e.message
|
119
132
|
end
|
120
|
-
unless
|
121
|
-
|
122
|
-
|
123
|
-
end
|
124
|
-
next unless listening_to_queue? job_info["q"]
|
125
|
-
|
126
|
-
scheduled_at = job_info["t"].to_f
|
127
|
-
@provider.update_earliest_scheduled_at(scheduled_at)
|
133
|
+
next unless job_info && listening_to_queue?(job_info["q"])
|
134
|
+
|
135
|
+
@provider.update_earliest_scheduled_at(job_info["t"].to_f)
|
128
136
|
end
|
129
137
|
end
|
130
138
|
end
|
131
139
|
|
132
|
-
# Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is
|
133
|
-
# Closes the connection after yielding it to the given block.
|
140
|
+
# Gets a DB connection and removes it from the pool. Sets the application name if +set_db_connection_name+ is
|
141
|
+
# true. Closes the connection after yielding it to the given block.
|
134
142
|
# (Grabbed from PG adapter for action cable)
|
135
143
|
# @yield yields the connection
|
136
144
|
# @yieldparam connection [PG::Connection] the DB connection
|
@@ -159,6 +167,7 @@ module Exekutor
|
|
159
167
|
raise UnsupportedDatabase,
|
160
168
|
"The raw connection of the active record connection adapter must be an instance of PG::Connection"
|
161
169
|
end
|
170
|
+
true
|
162
171
|
end
|
163
172
|
|
164
173
|
# For testing purposes
|
@@ -166,10 +175,28 @@ module Exekutor
|
|
166
175
|
@listening.true?
|
167
176
|
end
|
168
177
|
|
178
|
+
# Parses a NOTIFY payload to a job
|
179
|
+
class JobParser
|
180
|
+
JOB_INFO_KEYS = %w[id q t].freeze
|
181
|
+
|
182
|
+
def self.parse(payload)
|
183
|
+
job_info = begin
|
184
|
+
payload.split(";").to_h { |el| el.split(":") }
|
185
|
+
rescue StandardError
|
186
|
+
raise Error, "Invalid notification payload: #{payload}"
|
187
|
+
end
|
188
|
+
if (missing_keys = JOB_INFO_KEYS.select { |n| job_info[n].blank? }).present?
|
189
|
+
raise Error, "[Listener] Notification payload is missing #{missing_keys.join(", ")}"
|
190
|
+
end
|
191
|
+
|
192
|
+
job_info
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
169
196
|
# Raised when an error occurs in the listener.
|
170
197
|
class Error < Exekutor::Error; end
|
171
198
|
|
172
|
-
# Raised when the database connection is not an instance of PG::Connection
|
199
|
+
# Raised when the database connection is not an instance of +PG::Connection+.
|
173
200
|
class UnsupportedDatabase < Exekutor::Error; end
|
174
201
|
end
|
175
202
|
end
|
@@ -1,5 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "rainbow"
|
2
4
|
|
5
|
+
# The Exekutor namespace
|
3
6
|
module Exekutor
|
4
7
|
# @private
|
5
8
|
module Internal
|
@@ -9,7 +12,7 @@ module Exekutor
|
|
9
12
|
|
10
13
|
included do
|
11
14
|
# The log tags to use when writing to the log
|
12
|
-
mattr_accessor :log_tags, default: [
|
15
|
+
mattr_accessor :log_tags, default: [name.demodulize]
|
13
16
|
end
|
14
17
|
|
15
18
|
protected
|
@@ -35,25 +38,24 @@ module Exekutor
|
|
35
38
|
# Prints a message to STDOUT, unless {Exekutor::Configuration#quiet?} is true
|
36
39
|
# @private
|
37
40
|
def self.say(*args)
|
38
|
-
puts(*args) unless config.quiet?
|
41
|
+
puts(*args) unless config.quiet? # rubocop:disable Rails/Output
|
39
42
|
end
|
40
43
|
|
41
44
|
# Prints the error in the log and to STDERR (unless {Exekutor::Configuration#quiet?} is true)
|
42
|
-
# @param
|
45
|
+
# @param error [Exception] the error to print
|
43
46
|
# @param message [String] the message to print above the error
|
44
47
|
# @return [void]
|
45
|
-
def self.print_error(
|
46
|
-
|
47
|
-
error = "#{err.class} – #{err.message}\nat #{@backtrace_cleaner.clean(err.backtrace).join("\n ")}"
|
48
|
-
|
48
|
+
def self.print_error(error, message = nil)
|
49
|
+
error = strferr(error)
|
49
50
|
unless config.quiet?
|
50
|
-
|
51
|
-
|
51
|
+
warn Rainbow(message).bright.red if message
|
52
|
+
warn Rainbow(error).red
|
52
53
|
end
|
53
|
-
|
54
|
+
if config.quiet? || !ActiveSupport::Logger.logger_outputs_to?(logger, $stdout)
|
54
55
|
logger.error message if message
|
55
56
|
logger.error error
|
56
57
|
end
|
58
|
+
nil
|
57
59
|
end
|
58
60
|
|
59
61
|
# Gets the logger
|
@@ -71,4 +73,20 @@ module Exekutor
|
|
71
73
|
ActiveSupport::TaggedLogging.new logger
|
72
74
|
end
|
73
75
|
end
|
76
|
+
|
77
|
+
# @return [ActiveSupport::BacktraceCleaner] A cleaner for error backtraces
|
78
|
+
def self.backtrace_cleaner
|
79
|
+
@backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
|
80
|
+
end
|
81
|
+
|
82
|
+
# @return [String] The given error class, message, and cleaned backtrace as a string
|
83
|
+
def self.strferr(err)
|
84
|
+
raise ArgumentError, "err must not be nil" if err.nil?
|
85
|
+
|
86
|
+
"#{err.class} – #{err.message}\nat #{
|
87
|
+
err.backtrace ? backtrace_cleaner.clean(err.backtrace).join("\n ") : "unknown location"
|
88
|
+
}"
|
89
|
+
end
|
90
|
+
|
91
|
+
private_class_method :backtrace_cleaner, :strferr
|
74
92
|
end
|