exekutor 0.1.0 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -3
- data/exe/exekutor +2 -2
- data/lib/active_job/queue_adapters/exekutor_adapter.rb +2 -1
- data/lib/exekutor/asynchronous.rb +143 -75
- data/lib/exekutor/cleanup.rb +27 -28
- data/lib/exekutor/configuration.rb +102 -48
- data/lib/exekutor/hook.rb +15 -11
- data/lib/exekutor/info/worker.rb +3 -3
- data/lib/exekutor/internal/base_record.rb +2 -1
- data/lib/exekutor/internal/callbacks.rb +55 -35
- data/lib/exekutor/internal/cli/app.rb +33 -23
- data/lib/exekutor/internal/cli/application_loader.rb +17 -6
- data/lib/exekutor/internal/cli/cleanup.rb +54 -40
- data/lib/exekutor/internal/cli/daemon.rb +9 -11
- data/lib/exekutor/internal/cli/default_option_value.rb +3 -1
- data/lib/exekutor/internal/cli/info.rb +117 -84
- data/lib/exekutor/internal/cli/manager.rb +234 -123
- data/lib/exekutor/internal/configuration_builder.rb +49 -30
- data/lib/exekutor/internal/database_connection.rb +6 -0
- data/lib/exekutor/internal/executable.rb +12 -7
- data/lib/exekutor/internal/executor.rb +50 -21
- data/lib/exekutor/internal/hooks.rb +11 -8
- data/lib/exekutor/internal/listener.rb +85 -43
- data/lib/exekutor/internal/logger.rb +29 -10
- data/lib/exekutor/internal/provider.rb +96 -77
- data/lib/exekutor/internal/reserver.rb +66 -19
- data/lib/exekutor/internal/status_server.rb +87 -54
- data/lib/exekutor/job.rb +1 -1
- data/lib/exekutor/job_error.rb +1 -1
- data/lib/exekutor/job_options.rb +22 -13
- data/lib/exekutor/plugins/appsignal.rb +7 -5
- data/lib/exekutor/plugins.rb +8 -4
- data/lib/exekutor/queue.rb +69 -30
- data/lib/exekutor/version.rb +1 -1
- data/lib/exekutor/worker.rb +89 -48
- data/lib/exekutor.rb +2 -2
- data/lib/generators/exekutor/configuration_generator.rb +11 -6
- data/lib/generators/exekutor/install_generator.rb +24 -15
- data/lib/generators/exekutor/templates/install/functions/exekutor_broadcast_job_enqueued.sql +10 -0
- data/lib/generators/exekutor/templates/install/functions/exekutor_requeue_orphaned_jobs.sql +11 -0
- data/lib/generators/exekutor/templates/install/migrations/create_exekutor_schema.rb.erb +23 -22
- data/lib/generators/exekutor/templates/install/triggers/exekutor_broadcast_job_enqueued.sql +7 -0
- data/lib/generators/exekutor/templates/install/triggers/exekutor_requeue_orphaned_jobs.sql +5 -0
- data.tar.gz.sig +0 -0
- metadata +67 -23
- metadata.gz.sig +0 -0
- data/lib/generators/exekutor/templates/install/functions/job_notifier.sql +0 -7
- data/lib/generators/exekutor/templates/install/functions/requeue_orphaned_jobs.sql +0 -7
- data/lib/generators/exekutor/templates/install/triggers/notify_workers.sql +0 -6
- data/lib/generators/exekutor/templates/install/triggers/requeue_orphaned_jobs.sql +0 -5
@@ -1,5 +1,8 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require "rainbow"
|
2
4
|
|
5
|
+
# The Exekutor namespace
|
3
6
|
module Exekutor
|
4
7
|
# @private
|
5
8
|
module Internal
|
@@ -9,7 +12,7 @@ module Exekutor
|
|
9
12
|
|
10
13
|
included do
|
11
14
|
# The log tags to use when writing to the log
|
12
|
-
mattr_accessor :log_tags, default: [
|
15
|
+
mattr_accessor :log_tags, default: [name.demodulize]
|
13
16
|
end
|
14
17
|
|
15
18
|
protected
|
@@ -35,25 +38,24 @@ module Exekutor
|
|
35
38
|
# Prints a message to STDOUT, unless {Exekutor::Configuration#quiet?} is true
|
36
39
|
# @private
|
37
40
|
def self.say(*args)
|
38
|
-
puts(*args) unless config.quiet?
|
41
|
+
puts(*args) unless config.quiet? # rubocop:disable Rails/Output
|
39
42
|
end
|
40
43
|
|
41
44
|
# Prints the error in the log and to STDERR (unless {Exekutor::Configuration#quiet?} is true)
|
42
|
-
# @param
|
45
|
+
# @param error [Exception] the error to print
|
43
46
|
# @param message [String] the message to print above the error
|
44
47
|
# @return [void]
|
45
|
-
def self.print_error(
|
46
|
-
|
47
|
-
error = "#{err.class} – #{err.message}\nat #{@backtrace_cleaner.clean(err.backtrace).join("\n ")}"
|
48
|
-
|
48
|
+
def self.print_error(error, message = nil)
|
49
|
+
error = strferr(error)
|
49
50
|
unless config.quiet?
|
50
|
-
|
51
|
-
|
51
|
+
warn Rainbow(message).bright.red if message
|
52
|
+
warn Rainbow(error).red
|
52
53
|
end
|
53
|
-
|
54
|
+
if config.quiet? || !ActiveSupport::Logger.logger_outputs_to?(logger, $stdout)
|
54
55
|
logger.error message if message
|
55
56
|
logger.error error
|
56
57
|
end
|
58
|
+
nil
|
57
59
|
end
|
58
60
|
|
59
61
|
# Gets the logger
|
@@ -71,4 +73,21 @@ module Exekutor
|
|
71
73
|
ActiveSupport::TaggedLogging.new logger
|
72
74
|
end
|
73
75
|
end
|
76
|
+
|
77
|
+
# @return [ActiveSupport::BacktraceCleaner] A cleaner for error backtraces
|
78
|
+
def self.backtrace_cleaner
|
79
|
+
@backtrace_cleaner ||= ActiveSupport::BacktraceCleaner.new
|
80
|
+
end
|
81
|
+
|
82
|
+
# @return [String] The given error class, message, and cleaned backtrace as a string
|
83
|
+
def self.strferr(err)
|
84
|
+
raise ArgumentError, "err must not be nil" if err.nil?
|
85
|
+
return err if err.is_a? String
|
86
|
+
|
87
|
+
"#{err.class} – #{err.message}\nat #{
|
88
|
+
err.backtrace ? backtrace_cleaner.clean(err.backtrace).join("\n ") : "unknown location"
|
89
|
+
}"
|
90
|
+
end
|
91
|
+
|
92
|
+
private_class_method :backtrace_cleaner, :strferr
|
74
93
|
end
|
@@ -8,31 +8,33 @@ module Exekutor
|
|
8
8
|
module Internal
|
9
9
|
# Reserves jobs and provides them to an executor
|
10
10
|
class Provider
|
11
|
-
include
|
11
|
+
include Logger
|
12
|
+
include Callbacks
|
13
|
+
include Executable
|
12
14
|
|
13
15
|
define_callbacks :on_queue_empty, freeze: true
|
14
16
|
|
15
17
|
# Represents an unknown value
|
16
18
|
UNKNOWN = Object.new.freeze
|
17
|
-
private_constant
|
19
|
+
private_constant :UNKNOWN
|
18
20
|
|
19
21
|
MAX_WAIT_TIMEOUT = 300
|
20
|
-
private_constant
|
22
|
+
private_constant :MAX_WAIT_TIMEOUT
|
21
23
|
|
22
24
|
# Creates a new provider
|
23
25
|
# @param reserver [Reserver] the job reserver
|
24
26
|
# @param executor [Executor] the job executor
|
25
27
|
# @param pool [ThreadPoolExecutor] the thread pool to use
|
26
|
-
# @param polling_interval [
|
28
|
+
# @param polling_interval [ActiveSupport::Duration] the polling interval
|
27
29
|
# @param interval_jitter [Float] the polling interval jitter
|
28
|
-
def initialize(reserver:, executor:, pool:, polling_interval: 60,
|
30
|
+
def initialize(reserver:, executor:, pool:, polling_interval: 60.seconds,
|
29
31
|
interval_jitter: polling_interval.to_i > 1 ? polling_interval * 0.1 : 0)
|
30
32
|
super()
|
31
33
|
@reserver = reserver
|
32
34
|
@executor = executor
|
33
35
|
@pool = pool
|
34
36
|
|
35
|
-
@polling_interval = polling_interval.freeze
|
37
|
+
@polling_interval = polling_interval.to_i.freeze
|
36
38
|
@interval_jitter = interval_jitter.to_f.freeze
|
37
39
|
|
38
40
|
@event = Concurrent::Event.new
|
@@ -48,14 +50,14 @@ module Exekutor
|
|
48
50
|
|
49
51
|
# Always poll at startup to fill up threads, use small jitter so workers started at the same time dont hit
|
50
52
|
# the db at the same time
|
51
|
-
@next_poll_at.set (1 + 2 * Kernel.rand).
|
53
|
+
@next_poll_at.set (1 + (2 * Kernel.rand)).seconds.from_now.to_f
|
52
54
|
start_thread
|
53
55
|
true
|
54
56
|
end
|
55
57
|
|
56
58
|
# Stops the provider
|
57
59
|
def stop
|
58
|
-
|
60
|
+
self.state = :stopped
|
59
61
|
@event.set
|
60
62
|
end
|
61
63
|
|
@@ -63,7 +65,7 @@ module Exekutor
|
|
63
65
|
def poll
|
64
66
|
raise Exekutor::Error, "Provider is not running" unless running?
|
65
67
|
|
66
|
-
@next_poll_at.set Time.now
|
68
|
+
@next_poll_at.set Time.now.to_f
|
67
69
|
@event.set
|
68
70
|
end
|
69
71
|
|
@@ -71,51 +73,39 @@ module Exekutor
|
|
71
73
|
# argument is given. Updates the timestamp for the earliest job is a timestamp is given and that timestamp is
|
72
74
|
# before the known timestamp. Does nothing if a timestamp is given and the earliest job timestamp is not known.
|
73
75
|
# @param scheduled_at [Time,Numeric] the time a job is scheduled at
|
74
|
-
# @return [
|
76
|
+
# @return [Float,nil] the timestamp for the next job, or +nil+ if the timestamp is unknown or no jobs are pending
|
75
77
|
def update_earliest_scheduled_at(scheduled_at = UNKNOWN)
|
76
|
-
|
77
|
-
|
78
|
-
when UNKNOWN
|
79
|
-
# If we fetch the value from the DB, we can safely overwrite the UNKNOWN value
|
80
|
-
overwrite_unknown = true
|
81
|
-
scheduled_at = @reserver.earliest_scheduled_at
|
82
|
-
when Numeric
|
83
|
-
scheduled_at = Time.at(scheduled_at)
|
84
|
-
when Time
|
85
|
-
# All good
|
86
|
-
else
|
78
|
+
scheduled_at = scheduled_at.to_f if scheduled_at.is_a? Time
|
79
|
+
unless scheduled_at == UNKNOWN || scheduled_at.is_a?(Numeric)
|
87
80
|
raise ArgumentError, "scheduled_at must be a Time or Numeric"
|
88
81
|
end
|
89
82
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
scheduled_at
|
96
|
-
else
|
97
|
-
current
|
98
|
-
end
|
99
|
-
elsif current.nil? || scheduled_at.nil? || current > scheduled_at
|
100
|
-
updated = true
|
101
|
-
scheduled_at
|
102
|
-
else
|
103
|
-
current
|
104
|
-
end
|
83
|
+
changed = false
|
84
|
+
earliest_scheduled_at = @next_job_scheduled_at.update do |current|
|
85
|
+
earliest = ScheduledAtComparer.determine_earliest(current, scheduled_at, @reserver)
|
86
|
+
changed = earliest != current
|
87
|
+
earliest
|
105
88
|
end
|
106
|
-
|
89
|
+
|
90
|
+
if earliest_scheduled_at == UNKNOWN
|
107
91
|
nil
|
108
92
|
else
|
109
|
-
@event.set if
|
110
|
-
|
93
|
+
@event.set if changed && earliest_scheduled_at.present?
|
94
|
+
earliest_scheduled_at
|
111
95
|
end
|
112
96
|
end
|
113
97
|
|
114
98
|
private
|
115
99
|
|
116
100
|
# Starts the provision thread
|
117
|
-
def start_thread
|
118
|
-
|
101
|
+
def start_thread(delay: nil)
|
102
|
+
return unless running?
|
103
|
+
|
104
|
+
if delay
|
105
|
+
Concurrent::ScheduledTask.execute(delay, executor: @pool) { run }
|
106
|
+
else
|
107
|
+
@pool.post { run }
|
108
|
+
end
|
119
109
|
end
|
120
110
|
|
121
111
|
# Does the provisioning of jobs to the executor. Blocks until the provider is stopped.
|
@@ -134,19 +124,24 @@ module Exekutor
|
|
134
124
|
consecutive_errors.value = 0
|
135
125
|
end
|
136
126
|
end
|
137
|
-
rescue StandardError =>
|
138
|
-
|
139
|
-
consecutive_errors.increment
|
140
|
-
if running?
|
141
|
-
delay = restart_delay
|
142
|
-
logger.info "Restarting in %0.1f seconds…" % [delay]
|
143
|
-
Concurrent::ScheduledTask.execute(delay, executor: @pool, &method(:run))
|
144
|
-
end
|
127
|
+
rescue StandardError => e
|
128
|
+
on_thread_error(e)
|
145
129
|
ensure
|
146
130
|
BaseRecord.connection_pool.release_connection
|
147
131
|
@thread_running.make_false
|
148
132
|
end
|
149
133
|
|
134
|
+
# Called when an error is raised in #run
|
135
|
+
def on_thread_error(error)
|
136
|
+
Exekutor.on_fatal_error error, "[Provider] Runtime error!"
|
137
|
+
return unless running?
|
138
|
+
|
139
|
+
consecutive_errors.increment
|
140
|
+
delay = restart_delay
|
141
|
+
logger.info format("Restarting in %0.1f seconds…", delay)
|
142
|
+
start_thread delay: delay
|
143
|
+
end
|
144
|
+
|
150
145
|
# Waits for any event to happen. An event could be:
|
151
146
|
# - The listener was notified of a new job;
|
152
147
|
# - The next job is scheduled for the current time;
|
@@ -157,8 +152,8 @@ module Exekutor
|
|
157
152
|
return unless timeout.positive?
|
158
153
|
|
159
154
|
@event.wait timeout
|
160
|
-
rescue StandardError =>
|
161
|
-
Exekutor.on_fatal_error
|
155
|
+
rescue StandardError => e
|
156
|
+
Exekutor.on_fatal_error e, "[Provider] An error occurred while waiting"
|
162
157
|
sleep 0.1 if running?
|
163
158
|
ensure
|
164
159
|
throw :shutdown unless running?
|
@@ -171,21 +166,8 @@ module Exekutor
|
|
171
166
|
return unless available_workers.positive?
|
172
167
|
|
173
168
|
jobs = @reserver.reserve available_workers
|
174
|
-
unless jobs.nil?
|
175
|
-
|
176
|
-
logger.debug "Reserved #{jobs.size} job(s)"
|
177
|
-
jobs.each(&@executor.method(:post))
|
178
|
-
rescue Exception # rubocop:disable Lint/RescueException
|
179
|
-
# Try to release all jobs before re-raising
|
180
|
-
begin
|
181
|
-
Exekutor::Job.where(id: jobs.collect { |job| job[:id] }, status: "e")
|
182
|
-
.update_all(status: "p", worker_id: nil)
|
183
|
-
rescue # rubocop:disable Lint/RescueStandardError
|
184
|
-
# ignored
|
185
|
-
end
|
186
|
-
raise
|
187
|
-
end
|
188
|
-
end
|
169
|
+
execute_jobs jobs unless jobs.nil?
|
170
|
+
|
189
171
|
if jobs.nil? || jobs.size.to_i < available_workers
|
190
172
|
# If we ran out of work, update the earliest scheduled at
|
191
173
|
update_earliest_scheduled_at
|
@@ -194,8 +176,24 @@ module Exekutor
|
|
194
176
|
|
195
177
|
elsif @next_job_scheduled_at.get == UNKNOWN
|
196
178
|
# If the next job timestamp is still unknown, set it to now to indicate there's still work to do
|
197
|
-
@next_job_scheduled_at.set Time.now
|
179
|
+
@next_job_scheduled_at.set Time.now.to_f
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
# Posts the given jobs to the executor thread pool
|
184
|
+
# @param jobs [Array] the jobs to execute
|
185
|
+
def execute_jobs(jobs)
|
186
|
+
logger.debug "Reserved #{jobs.size} job(s)"
|
187
|
+
jobs.each { |job| @executor.post(job) }
|
188
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
189
|
+
# Try to release all jobs before re-raising
|
190
|
+
begin
|
191
|
+
Exekutor::Job.where(id: jobs.pluck(:id), status: "e")
|
192
|
+
.update_all(status: "p", worker_id: nil)
|
193
|
+
rescue StandardError
|
194
|
+
# ignored
|
198
195
|
end
|
196
|
+
raise
|
199
197
|
end
|
200
198
|
|
201
199
|
def perform_pending_job_updates
|
@@ -224,7 +222,7 @@ module Exekutor
|
|
224
222
|
return if jobs&.size.to_i.zero?
|
225
223
|
|
226
224
|
logger.info "Restarting #{jobs.size} abandoned job#{"s" if jobs.size > 1}"
|
227
|
-
jobs.each
|
225
|
+
jobs.each { |job| @executor.post(job) }
|
228
226
|
end
|
229
227
|
|
230
228
|
# @return [Boolean] Whether the polling is enabled. Ie. whether a polling interval is set.
|
@@ -232,7 +230,7 @@ module Exekutor
|
|
232
230
|
@polling_interval.present?
|
233
231
|
end
|
234
232
|
|
235
|
-
# @return [
|
233
|
+
# @return [Float,nil] the 'scheduled at' value for the next job, or nil if unknown or if there is no pending job
|
236
234
|
def next_job_scheduled_at
|
237
235
|
at = @next_job_scheduled_at.get
|
238
236
|
if at == UNKNOWN
|
@@ -243,10 +241,10 @@ module Exekutor
|
|
243
241
|
end
|
244
242
|
end
|
245
243
|
|
246
|
-
# @return [
|
244
|
+
# @return [Float,nil] When the next poll is scheduled, or nil if polling is disabled
|
247
245
|
def next_poll_scheduled_at
|
248
246
|
if polling_enabled?
|
249
|
-
@next_poll_at.update { |planned_at| planned_at || Time.now + polling_interval }
|
247
|
+
@next_poll_at.update { |planned_at| planned_at || (Time.now.to_f + polling_interval) }
|
250
248
|
else
|
251
249
|
# noinspection RubyMismatchedReturnType
|
252
250
|
@next_poll_at.get
|
@@ -262,9 +260,9 @@ module Exekutor
|
|
262
260
|
|
263
261
|
timeout = [MAX_WAIT_TIMEOUT].tap do |timeouts|
|
264
262
|
# noinspection RubyMismatchedArgumentType
|
265
|
-
timeouts.append next_job_at - Time.now if next_job_at
|
263
|
+
timeouts.append next_job_at - Time.now.to_f if next_job_at
|
266
264
|
# noinspection RubyMismatchedArgumentType
|
267
|
-
timeouts.append next_poll_at - Time.now if next_poll_at
|
265
|
+
timeouts.append next_poll_at - Time.now.to_f if next_poll_at
|
268
266
|
end.min
|
269
267
|
|
270
268
|
if timeout <= 0.001
|
@@ -278,13 +276,13 @@ module Exekutor
|
|
278
276
|
# @return [Boolean] Whether the `reserver` should be called.
|
279
277
|
def reserve_jobs_now?
|
280
278
|
next_poll_at = next_poll_scheduled_at
|
281
|
-
if next_poll_at && next_poll_at - Time.now <= 0.001
|
282
|
-
@next_poll_at.update { Time.now + polling_interval if polling_enabled? }
|
279
|
+
if next_poll_at && next_poll_at - Time.now.to_f <= 0.001
|
280
|
+
@next_poll_at.update { Time.now.to_f + polling_interval if polling_enabled? }
|
283
281
|
return true
|
284
282
|
end
|
285
283
|
|
286
284
|
next_job_at = next_job_scheduled_at
|
287
|
-
next_job_at && next_job_at <= Time.now
|
285
|
+
next_job_at && next_job_at <= Time.now.to_f
|
288
286
|
end
|
289
287
|
|
290
288
|
# @return [Float] Gets the polling interval jitter
|
@@ -295,7 +293,7 @@ module Exekutor
|
|
295
293
|
# Get the polling interval. If a jitter is configured, the interval is reduced or increased by `0.5 * jitter`.
|
296
294
|
# @return [Float] The amount of seconds before the next poll
|
297
295
|
def polling_interval
|
298
|
-
raise "Polling is disabled"
|
296
|
+
raise "Polling is disabled" if @polling_interval.blank?
|
299
297
|
|
300
298
|
@polling_interval + if polling_interval_jitter.zero?
|
301
299
|
0
|
@@ -303,6 +301,27 @@ module Exekutor
|
|
303
301
|
(Kernel.rand - 0.5) * polling_interval_jitter
|
304
302
|
end
|
305
303
|
end
|
304
|
+
|
305
|
+
# Abstraction to determine earliest out of 2 scheduled at timestamps
|
306
|
+
class ScheduledAtComparer
|
307
|
+
def self.determine_earliest(current_value, other_value, reserver)
|
308
|
+
if other_value == UNKNOWN
|
309
|
+
# Fetch the value from the db
|
310
|
+
reserver.earliest_scheduled_at&.to_f
|
311
|
+
elsif current_value == UNKNOWN
|
312
|
+
# We don't know the value yet, don't use the given value as earliest unless work is scheduled immediately
|
313
|
+
other_value > Time.now.to_f ? UNKNOWN : other_value
|
314
|
+
elsif current_value.nil? || current_value > other_value
|
315
|
+
# The given value is earlier than the known value
|
316
|
+
other_value
|
317
|
+
else
|
318
|
+
# No changes
|
319
|
+
current_value
|
320
|
+
end
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
private_constant :ScheduledAtComparer
|
306
325
|
end
|
307
326
|
end
|
308
327
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
module Exekutor
|
3
4
|
# @private
|
4
5
|
module Internal
|
@@ -10,23 +11,23 @@ module Exekutor
|
|
10
11
|
# Creates a new Reserver
|
11
12
|
# @param worker_id [String] the id of the worker
|
12
13
|
# @param queues [Array<String>] the queues to watch
|
13
|
-
def initialize(worker_id, queues)
|
14
|
+
def initialize(worker_id, queues: nil, min_priority: nil, max_priority: nil)
|
14
15
|
@worker_id = worker_id
|
15
|
-
@
|
16
|
+
@reserve_filter_sql = build_filter_sql(queues: queues, min_priority: min_priority, max_priority: max_priority)
|
16
17
|
@json_serializer = Exekutor.config.load_json_serializer
|
17
18
|
end
|
18
19
|
|
19
20
|
# Reserves pending jobs
|
20
21
|
# @param limit [Integer] the number of jobs to reserve
|
21
|
-
# @return [Array<
|
22
|
+
# @return [Array<Hash>,nil] the reserved jobs, or nil if no jobs were reserved
|
22
23
|
def reserve(limit)
|
23
24
|
return unless limit.positive?
|
24
25
|
|
25
26
|
results = Exekutor::Job.connection.exec_query <<~SQL, ACTION_NAME, [@worker_id, limit], prepare: true
|
26
27
|
UPDATE exekutor_jobs SET worker_id = $1, status = 'e' WHERE id IN (
|
27
28
|
SELECT id FROM exekutor_jobs
|
28
|
-
WHERE scheduled_at <= now() AND "status"='p' #{@
|
29
|
-
ORDER BY priority, scheduled_at, enqueued_at
|
29
|
+
WHERE scheduled_at <= now() AND "status"='p'#{" AND #{@reserve_filter_sql}" if @reserve_filter_sql}
|
30
|
+
ORDER BY priority#{" DESC" if Exekutor.config.inverse_priority?}, scheduled_at, enqueued_at
|
30
31
|
FOR UPDATE SKIP LOCKED
|
31
32
|
LIMIT $2
|
32
33
|
) RETURNING "id", "payload", "options", "scheduled_at"
|
@@ -36,6 +37,9 @@ module Exekutor
|
|
36
37
|
parse_jobs results
|
37
38
|
end
|
38
39
|
|
40
|
+
# Gets the jobs that are assigned to this worker and have an id that is not included in +active_job_ids+
|
41
|
+
# @param active_job_ids [Array<String>] The ids of the jobs that should be excluded
|
42
|
+
# @return [Array<Hash>] the jobs
|
39
43
|
def get_abandoned_jobs(active_job_ids)
|
40
44
|
jobs = Exekutor::Job.executing.where(worker_id: @worker_id)
|
41
45
|
jobs = jobs.where.not(id: active_job_ids) if active_job_ids.present?
|
@@ -47,7 +51,7 @@ module Exekutor
|
|
47
51
|
# @return [Time,nil] The earliest scheduled at, or nil if the queues are empty
|
48
52
|
def earliest_scheduled_at
|
49
53
|
jobs = Exekutor::Job.pending
|
50
|
-
jobs.where! @
|
54
|
+
jobs.where! @reserve_filter_sql unless @reserve_filter_sql.nil?
|
51
55
|
jobs.minimum(:scheduled_at)
|
52
56
|
end
|
53
57
|
|
@@ -59,7 +63,7 @@ module Exekutor
|
|
59
63
|
{ id: result["id"],
|
60
64
|
payload: parse_json(result["payload"]),
|
61
65
|
options: parse_json(result["options"]),
|
62
|
-
scheduled_at: result[
|
66
|
+
scheduled_at: result["scheduled_at"] }
|
63
67
|
end
|
64
68
|
end
|
65
69
|
|
@@ -68,28 +72,71 @@ module Exekutor
|
|
68
72
|
@json_serializer.load str unless str.nil?
|
69
73
|
end
|
70
74
|
|
75
|
+
# Builds SQL filter for the given queues and priorities
|
76
|
+
def build_filter_sql(queues:, min_priority:, max_priority:)
|
77
|
+
filters = [
|
78
|
+
build_queue_filter_sql(queues),
|
79
|
+
build_priority_filter_sql(min_priority, max_priority)
|
80
|
+
]
|
81
|
+
filters.compact!
|
82
|
+
filters.join(" AND ") unless filters.empty?
|
83
|
+
end
|
84
|
+
|
71
85
|
# Builds SQL filter for the given queues
|
72
86
|
def build_queue_filter_sql(queues)
|
73
87
|
return nil if queues.nil? || (queues.is_a?(Array) && queues.empty?)
|
74
|
-
unless queues.is_a?(String) || queues.is_a?(Symbol) || queues.is_a?(Array)
|
75
|
-
raise ArgumentError, "queues must be nil, a String, Symbol, or an array of Strings or Symbols"
|
76
|
-
end
|
77
88
|
|
78
89
|
queues = queues.first if queues.is_a?(Array) && queues.one?
|
79
|
-
|
80
|
-
unless queues.all? { |q| (q.is_a?(String) || q.is_a?(Symbol)) && !q.blank? }
|
81
|
-
raise ArgumentError, "queues contains an invalid value"
|
82
|
-
end
|
90
|
+
validate_queues! queues
|
83
91
|
|
84
|
-
|
85
|
-
|
86
|
-
|
92
|
+
conditions = if queues.is_a? Array
|
93
|
+
["queue IN (?)", queues]
|
94
|
+
else
|
95
|
+
["queue = ?", queues]
|
96
|
+
end
|
97
|
+
Exekutor::Job.sanitize_sql_for_conditions conditions
|
98
|
+
end
|
99
|
+
|
100
|
+
# Builds SQL filter for the given priorities
|
101
|
+
def build_priority_filter_sql(minimum, maximum)
|
102
|
+
minimum = coerce_priority(minimum)
|
103
|
+
maximum = coerce_priority(maximum)
|
87
104
|
|
88
|
-
|
105
|
+
conditions = if minimum && maximum
|
106
|
+
["priority BETWEEN ? AND ?", minimum, maximum]
|
107
|
+
elsif minimum
|
108
|
+
["priority >= ?", minimum]
|
109
|
+
elsif maximum
|
110
|
+
["priority <= ?", maximum]
|
111
|
+
end
|
112
|
+
Exekutor::Job.sanitize_sql_for_conditions conditions if conditions
|
113
|
+
end
|
114
|
+
|
115
|
+
# @return [Integer,nil] returns nil unless +priority+ is between 1 and 32,766
|
116
|
+
def coerce_priority(priority)
|
117
|
+
priority if priority && (1..32_766).cover?(priority)
|
118
|
+
end
|
119
|
+
|
120
|
+
# Raises an error if the queues value is invalid
|
121
|
+
# @param queues [String,Symbol,Array<String,Symbol>] the queues to validate
|
122
|
+
# @raise [ArgumentError] if the queue is invalid or includes an invalid value
|
123
|
+
def validate_queues!(queues)
|
124
|
+
case queues
|
125
|
+
when Array
|
126
|
+
raise ArgumentError, "queues contains an invalid value" unless queues.all? { |queue| valid_queue_name? queue }
|
127
|
+
when String, Symbol
|
128
|
+
raise ArgumentError, "queue name cannot be empty" unless valid_queue_name? queues
|
129
|
+
else
|
130
|
+
raise ArgumentError,
|
131
|
+
"queues must be nil, a String, Symbol, or an array of Strings or Symbols (Actual: #{queues.class})"
|
89
132
|
end
|
90
133
|
end
|
91
134
|
|
135
|
+
# @param queue [String,Symbol] the name of a queue
|
136
|
+
# @return [Boolean] whether the name is a valid queue name
|
137
|
+
def valid_queue_name?(queue)
|
138
|
+
(queue.is_a?(String) || queue.is_a?(Symbol)) && queue.present? && queue.length <= Queue::MAX_NAME_LENGTH
|
139
|
+
end
|
92
140
|
end
|
93
141
|
end
|
94
142
|
end
|
95
|
-
|