inst-jobs 3.0.8 → 3.1.11
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +4 -2
- data/db/migrate/20200825011002_add_strand_order_override.rb +2 -1
- data/db/migrate/20210809145804_add_n_strand_index.rb +2 -1
- data/db/migrate/20220328152900_add_failed_jobs_indicies.rb +12 -0
- data/db/migrate/20220519204546_add_requeued_job_id_to_failed_jobs.rb +7 -0
- data/lib/delayed/backend/active_record.rb +63 -6
- data/lib/delayed/backend/base.rb +28 -10
- data/lib/delayed/batch.rb +1 -1
- data/lib/delayed/cli.rb +3 -2
- data/lib/delayed/lifecycle.rb +9 -2
- data/lib/delayed/log_tailer.rb +2 -2
- data/lib/delayed/message_sending.rb +8 -5
- data/lib/delayed/performable_method.rb +22 -16
- data/lib/delayed/periodic.rb +2 -2
- data/lib/delayed/pool.rb +12 -2
- data/lib/delayed/server.rb +8 -2
- data/lib/delayed/settings.rb +5 -1
- data/lib/delayed/version.rb +1 -1
- data/lib/delayed/work_queue/parent_process/server.rb +7 -3
- data/lib/delayed/worker/health_check.rb +1 -1
- data/lib/delayed/worker/process_helper.rb +4 -4
- data/lib/delayed/worker.rb +8 -8
- metadata +16 -89
- data/spec/active_record_job_spec.rb +0 -294
- data/spec/delayed/cli_spec.rb +0 -25
- data/spec/delayed/daemon_spec.rb +0 -38
- data/spec/delayed/message_sending_spec.rb +0 -108
- data/spec/delayed/periodic_spec.rb +0 -32
- data/spec/delayed/server_spec.rb +0 -103
- data/spec/delayed/settings_spec.rb +0 -48
- data/spec/delayed/work_queue/in_process_spec.rb +0 -31
- data/spec/delayed/work_queue/parent_process/client_spec.rb +0 -87
- data/spec/delayed/work_queue/parent_process/server_spec.rb +0 -280
- data/spec/delayed/work_queue/parent_process_spec.rb +0 -60
- data/spec/delayed/worker/consul_health_check_spec.rb +0 -63
- data/spec/delayed/worker/health_check_spec.rb +0 -134
- data/spec/delayed/worker_spec.rb +0 -105
- data/spec/migrate/20140924140513_add_story_table.rb +0 -9
- data/spec/sample_jobs.rb +0 -79
- data/spec/shared/delayed_batch.rb +0 -105
- data/spec/shared/delayed_method.rb +0 -287
- data/spec/shared/performable_method.rb +0 -75
- data/spec/shared/shared_backend.rb +0 -1221
- data/spec/shared/testing.rb +0 -50
- data/spec/shared/worker.rb +0 -413
- data/spec/shared_jobs_specs.rb +0 -17
- data/spec/spec_helper.rb +0 -136
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 372437f25842f762a361453b2846ff561868f4f1783e91b090f2dfc40a8b22f6
|
4
|
+
data.tar.gz: '091128e8391c5acf7f12263887af7f79a7253ed721c39978cd1e1a3fbd93157c'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 17b6e974d17be8d6ccc6d7f01f2747789b37e48461c862666542957fb299cda415c9aecda8885110254c5d0b7ad45768f6f82c6d85856ab430717a0db89a0db5
|
7
|
+
data.tar.gz: 6e093a601037c58be6d04d96c9b2965099a084285cc7ce80908dfaad1e6515ddd007172b96d2e6e8e3d8f9702d7af4bd5dca82041fa362887ae3d170aed4c5ca
|
@@ -9,7 +9,8 @@ class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
|
|
9
9
|
|
10
10
|
def up
|
11
11
|
rename_index :delayed_jobs, "get_delayed_jobs_index", "get_delayed_jobs_index_old"
|
12
|
-
add_index :delayed_jobs,
|
12
|
+
add_index :delayed_jobs,
|
13
|
+
%i[queue priority run_at id],
|
13
14
|
algorithm: :concurrently,
|
14
15
|
where: "locked_at IS NULL AND next_in_strand",
|
15
16
|
name: "get_delayed_jobs_index"
|
@@ -18,7 +19,8 @@ class AddIdToGetDelayedJobsIndex < ActiveRecord::Migration[4.2]
|
|
18
19
|
|
19
20
|
def down
|
20
21
|
rename_index :delayed_jobs, "get_delayed_jobs_index", "get_delayed_jobs_index_old"
|
21
|
-
add_index :delayed_jobs,
|
22
|
+
add_index :delayed_jobs,
|
23
|
+
%i[priority run_at queue],
|
22
24
|
algorithm: :concurrently,
|
23
25
|
where: "locked_at IS NULL AND next_in_strand",
|
24
26
|
name: "get_delayed_jobs_index"
|
@@ -10,7 +10,8 @@ class AddStrandOrderOverride < ActiveRecord::Migration[4.2]
|
|
10
10
|
def up
|
11
11
|
add_column :delayed_jobs, :strand_order_override, :integer, default: 0, null: false
|
12
12
|
add_column :failed_jobs, :strand_order_override, :integer, default: 0, null: false
|
13
|
-
add_index :delayed_jobs,
|
13
|
+
add_index :delayed_jobs,
|
14
|
+
%i[strand strand_order_override id],
|
14
15
|
algorithm: :concurrently,
|
15
16
|
where: "strand IS NOT NULL",
|
16
17
|
name: "next_in_strand_index"
|
@@ -4,7 +4,8 @@ class AddNStrandIndex < ActiveRecord::Migration[5.2]
|
|
4
4
|
disable_ddl_transaction!
|
5
5
|
|
6
6
|
def change
|
7
|
-
add_index :delayed_jobs,
|
7
|
+
add_index :delayed_jobs,
|
8
|
+
%i[strand next_in_strand id],
|
8
9
|
name: "n_strand_index",
|
9
10
|
where: "strand IS NOT NULL",
|
10
11
|
algorithm: :concurrently
|
@@ -0,0 +1,12 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class AddFailedJobsIndicies < ActiveRecord::Migration[5.2]
|
4
|
+
disable_ddl_transaction!
|
5
|
+
|
6
|
+
def change
|
7
|
+
add_index :failed_jobs, :failed_at, algorithm: :concurrently
|
8
|
+
add_index :failed_jobs, :strand, where: "strand IS NOT NULL", algorithm: :concurrently
|
9
|
+
add_index :failed_jobs, :singleton, where: "singleton IS NOT NULL", algorithm: :concurrently
|
10
|
+
add_index :failed_jobs, :tag, algorithm: :concurrently
|
11
|
+
end
|
12
|
+
end
|
@@ -28,7 +28,11 @@ module Delayed
|
|
28
28
|
scope :next_in_strand_order, -> { order(:strand_order_override, :id) }
|
29
29
|
|
30
30
|
def self.reconnect!
|
31
|
-
|
31
|
+
if Rails.version < "6.1"
|
32
|
+
::ActiveRecord::Base.connection_handler.clear_all_connections!
|
33
|
+
else
|
34
|
+
::ActiveRecord::Base.connection_handler.clear_all_connections!(nil)
|
35
|
+
end
|
32
36
|
end
|
33
37
|
|
34
38
|
class << self
|
@@ -62,11 +66,21 @@ module Delayed
|
|
62
66
|
_write_attribute(column, current_time) unless attribute_present?(column)
|
63
67
|
end
|
64
68
|
|
65
|
-
attribute_names =
|
69
|
+
attribute_names = if Rails.version < "7.0"
|
70
|
+
attribute_names_for_partial_writes
|
71
|
+
else
|
72
|
+
attribute_names_for_partial_inserts
|
73
|
+
end
|
66
74
|
attribute_names = attributes_for_create(attribute_names)
|
67
75
|
values = attributes_with_values(attribute_names)
|
68
76
|
|
69
|
-
im =
|
77
|
+
im = if Rails.version < "7.0"
|
78
|
+
self.class.arel_table.compile_insert(self.class.send(:_substitute_values, values))
|
79
|
+
else
|
80
|
+
im = Arel::InsertManager.new(self.class.arel_table)
|
81
|
+
im.insert(values.transform_keys { |name| self.class.arel_table[name] })
|
82
|
+
im
|
83
|
+
end
|
70
84
|
|
71
85
|
lock_and_insert = values["strand"] && instance_of?(Job)
|
72
86
|
# can't use prepared statements if we're combining multiple statemenets
|
@@ -101,7 +115,8 @@ module Delayed
|
|
101
115
|
# but we don't need to lock when inserting into Delayed::Failed
|
102
116
|
if values["strand"] && instance_of?(Job)
|
103
117
|
fn_name = connection.quote_table_name("half_md5_as_bigint")
|
104
|
-
|
118
|
+
quoted_strand = connection.quote((Rails.version < "7.0") ? values["strand"] : values["strand"].value)
|
119
|
+
sql = "SELECT pg_advisory_xact_lock(#{fn_name}(#{quoted_strand})); #{sql}"
|
105
120
|
end
|
106
121
|
result = connection.execute(sql, "#{self.class} Create")
|
107
122
|
self.id = result.values.first&.first
|
@@ -238,7 +253,7 @@ module Delayed
|
|
238
253
|
offset = 0,
|
239
254
|
query = nil)
|
240
255
|
scope = scope_for_flavor(flavor, query)
|
241
|
-
order = flavor.to_s == "future" ? "run_at" : "id desc"
|
256
|
+
order = (flavor.to_s == "future") ? "run_at" : "id desc"
|
242
257
|
scope.order(order).limit(limit).offset(offset).to_a
|
243
258
|
end
|
244
259
|
|
@@ -305,6 +320,24 @@ module Delayed
|
|
305
320
|
scope.order(Arel.sql("COUNT(tag) DESC")).count.map { |t, c| { tag: t, count: c } }
|
306
321
|
end
|
307
322
|
|
323
|
+
# given a scope of non-stranded queued jobs, apply a temporary strand to throttle their execution
|
324
|
+
# returns [job_count, new_strand]
|
325
|
+
# (this is designed for use in a Rails console or the Canvas Jobs interface)
|
326
|
+
def self.apply_temp_strand!(job_scope, max_concurrent: 1)
|
327
|
+
if job_scope.where("strand IS NOT NULL OR singleton IS NOT NULL").exists?
|
328
|
+
raise ArgumentError, "can't apply strand to already stranded jobs"
|
329
|
+
end
|
330
|
+
|
331
|
+
job_count = 0
|
332
|
+
new_strand = "tmp_strand_#{SecureRandom.alphanumeric(16)}"
|
333
|
+
::Delayed::Job.transaction do
|
334
|
+
job_count = job_scope.update_all(strand: new_strand, max_concurrent: max_concurrent, next_in_strand: false)
|
335
|
+
::Delayed::Job.where(strand: new_strand).order(:id).limit(max_concurrent).update_all(next_in_strand: true)
|
336
|
+
end
|
337
|
+
|
338
|
+
[job_count, new_strand]
|
339
|
+
end
|
340
|
+
|
308
341
|
def self.maybe_silence_periodic_log(&block)
|
309
342
|
if Settings.silence_periodic_log
|
310
343
|
::ActiveRecord::Base.logger.silence(&block)
|
@@ -471,7 +504,7 @@ module Delayed
|
|
471
504
|
transaction do
|
472
505
|
# for db performance reasons, we only need one process doing this at a time
|
473
506
|
# so if we can't get an advisory lock, just abort. we'll try again soon
|
474
|
-
|
507
|
+
next unless attempt_advisory_lock(prefetch_jobs_lock_name)
|
475
508
|
|
476
509
|
horizon = db_time_now - (Settings.parent_process[:prefetched_jobs_timeout] * 4)
|
477
510
|
where("locked_by LIKE 'prefetch:%' AND locked_at<?", horizon).update_all(locked_at: nil, locked_by: nil)
|
@@ -568,6 +601,30 @@ module Delayed
|
|
568
601
|
class Failed < Job
|
569
602
|
include Delayed::Backend::Base
|
570
603
|
self.table_name = :failed_jobs
|
604
|
+
|
605
|
+
def self.cleanup_old_jobs(before_date, batch_size: 10_000)
|
606
|
+
where("failed_at < ?", before_date).in_batches(of: batch_size).delete_all
|
607
|
+
end
|
608
|
+
|
609
|
+
def requeue!
|
610
|
+
attrs = attributes.except("id",
|
611
|
+
"last_error",
|
612
|
+
"locked_at",
|
613
|
+
"failed_at",
|
614
|
+
"locked_by",
|
615
|
+
"original_job_id",
|
616
|
+
"requeued_job_id")
|
617
|
+
self.class.transaction do
|
618
|
+
job = nil
|
619
|
+
Delayed::Worker.lifecycle.run_callbacks(:create, attrs.merge("payload_object" => payload_object)) do
|
620
|
+
job = Job.create(attrs)
|
621
|
+
end
|
622
|
+
self.requeued_job_id = job.id
|
623
|
+
save!
|
624
|
+
JobTracking.job_created(job)
|
625
|
+
job
|
626
|
+
end
|
627
|
+
end
|
571
628
|
end
|
572
629
|
end
|
573
630
|
end
|
data/lib/delayed/backend/base.rb
CHANGED
@@ -62,6 +62,9 @@ module Delayed
|
|
62
62
|
kwargs[:singleton] = singleton
|
63
63
|
|
64
64
|
raise ArgumentError, "Only one of strand or n_strand can be used" if strand && n_strand
|
65
|
+
if (strand || n_strand) && run_at && run_at > Job.db_time_now + Settings.stranded_run_at_grace_period
|
66
|
+
raise ArgumentError, "Do not use run_at with strand; you may inadvertently clog the strand"
|
67
|
+
end
|
65
68
|
|
66
69
|
# If two parameters are given to n_strand, the first param is used
|
67
70
|
# as the strand name for looking up the Setting, while the second
|
@@ -86,8 +89,12 @@ module Delayed
|
|
86
89
|
kwargs.merge!(n_strand_options(full_strand_name, num_strands))
|
87
90
|
end
|
88
91
|
|
92
|
+
job = nil
|
93
|
+
|
89
94
|
if singleton
|
90
|
-
|
95
|
+
Delayed::Worker.lifecycle.run_callbacks(:create, kwargs) do
|
96
|
+
job = create(**kwargs)
|
97
|
+
end
|
91
98
|
elsif batches && strand.nil? && run_at.nil?
|
92
99
|
batch_enqueue_args = kwargs.slice(*self.batch_enqueue_args)
|
93
100
|
batches[batch_enqueue_args] << kwargs
|
@@ -95,7 +102,9 @@ module Delayed
|
|
95
102
|
else
|
96
103
|
raise ArgumentError, "on_conflict can only be provided with singleton" if kwargs[:on_conflict]
|
97
104
|
|
98
|
-
|
105
|
+
Delayed::Worker.lifecycle.run_callbacks(:create, kwargs) do
|
106
|
+
job = create(**kwargs)
|
107
|
+
end
|
99
108
|
end
|
100
109
|
|
101
110
|
JobTracking.job_created(job)
|
@@ -107,7 +116,7 @@ module Delayed
|
|
107
116
|
# effectively balancing the load during queueing
|
108
117
|
# overwritten in ActiveRecord::Job to use triggers to balance at run time
|
109
118
|
def n_strand_options(strand_name, num_strands)
|
110
|
-
strand_num = num_strands > 1 ? rand(num_strands) + 1 : 1
|
119
|
+
strand_num = (num_strands > 1) ? rand(num_strands) + 1 : 1
|
111
120
|
strand_name += ":#{strand_num}" if strand_num > 1
|
112
121
|
{ strand: strand_name }
|
113
122
|
end
|
@@ -208,7 +217,7 @@ module Delayed
|
|
208
217
|
def failed?
|
209
218
|
failed_at
|
210
219
|
end
|
211
|
-
|
220
|
+
alias_method :failed, :failed?
|
212
221
|
|
213
222
|
def expired?
|
214
223
|
expires_at && (self.class.db_time_now >= expires_at)
|
@@ -222,8 +231,7 @@ module Delayed
|
|
222
231
|
# Uses an exponential scale depending on the number of failed attempts.
|
223
232
|
def reschedule(error = nil, time = nil)
|
224
233
|
begin
|
225
|
-
|
226
|
-
return_code = obj.on_failure(error) if obj.respond_to?(:on_failure)
|
234
|
+
return_code = invoke_payload_object_cb(:on_failure, error)
|
227
235
|
rescue
|
228
236
|
# don't allow a failed deserialization to prevent rescheduling
|
229
237
|
end
|
@@ -245,8 +253,7 @@ module Delayed
|
|
245
253
|
def permanent_failure(error)
|
246
254
|
begin
|
247
255
|
# notify the payload_object of a permanent failure
|
248
|
-
|
249
|
-
obj.on_permanent_failure(error) if obj.respond_to?(:on_permanent_failure)
|
256
|
+
invoke_payload_object_cb(:on_permanent_failure, error)
|
250
257
|
rescue
|
251
258
|
# don't allow a failed deserialization to prevent destroying the job
|
252
259
|
end
|
@@ -263,7 +270,7 @@ module Delayed
|
|
263
270
|
end
|
264
271
|
|
265
272
|
def payload_object
|
266
|
-
@payload_object ||= deserialize(self["handler"]
|
273
|
+
@payload_object ||= deserialize(self["handler"])
|
267
274
|
end
|
268
275
|
|
269
276
|
def name
|
@@ -306,7 +313,13 @@ module Delayed
|
|
306
313
|
payload_object.perform
|
307
314
|
ensure
|
308
315
|
Delayed::Job.in_delayed_job = false
|
309
|
-
|
316
|
+
unless Rails.env.test?
|
317
|
+
if Rails.version < "6.1"
|
318
|
+
::ActiveRecord::Base.clear_active_connections!
|
319
|
+
else
|
320
|
+
::ActiveRecord::Base.clear_active_connections!(nil)
|
321
|
+
end
|
322
|
+
end
|
310
323
|
end
|
311
324
|
end
|
312
325
|
end
|
@@ -395,6 +408,11 @@ module Delayed
|
|
395
408
|
md[1].constantize
|
396
409
|
end
|
397
410
|
|
411
|
+
def invoke_payload_object_cb(method, ...)
|
412
|
+
obj = payload_object
|
413
|
+
obj.public_send(method, ...) if obj.respond_to?(method)
|
414
|
+
end
|
415
|
+
|
398
416
|
public
|
399
417
|
|
400
418
|
def initialize_defaults
|
data/lib/delayed/batch.rb
CHANGED
data/lib/delayed/cli.rb
CHANGED
@@ -49,7 +49,7 @@ module Delayed
|
|
49
49
|
daemon.daemonize!
|
50
50
|
start
|
51
51
|
when nil
|
52
|
-
puts option_parser.
|
52
|
+
puts option_parser.help
|
53
53
|
else
|
54
54
|
raise("Unknown command: #{command.inspect}")
|
55
55
|
end
|
@@ -82,7 +82,8 @@ module Delayed
|
|
82
82
|
opts.on("-c", "--config [CONFIG_PATH]", "Use alternate config file (default #{@options[:config_file]})") do |c|
|
83
83
|
@options[:config_file] = c
|
84
84
|
end
|
85
|
-
opts.on("-p",
|
85
|
+
opts.on("-p",
|
86
|
+
"--pid [PID_PATH]",
|
86
87
|
"Use alternate folder for PID files (default #{@options[:pid_folder]})") do |p|
|
87
88
|
@options[:pid_folder] = p
|
88
89
|
end
|
data/lib/delayed/lifecycle.rb
CHANGED
@@ -5,6 +5,7 @@ module Delayed
|
|
5
5
|
|
6
6
|
class Lifecycle
|
7
7
|
EVENTS = {
|
8
|
+
create: [:args],
|
8
9
|
error: %i[worker job exception],
|
9
10
|
exceptional_exit: %i[worker exception],
|
10
11
|
execute: [:worker],
|
@@ -45,7 +46,7 @@ module Delayed
|
|
45
46
|
missing_callback(event) unless @callbacks.key?(event)
|
46
47
|
|
47
48
|
unless EVENTS[event].size == args.size
|
48
|
-
raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(
|
49
|
+
raise ArgumentError, "Callback #{event} expects #{EVENTS[event].size} parameter(s): #{EVENTS[event].join(", ")}"
|
49
50
|
end
|
50
51
|
|
51
52
|
@callbacks[event].execute(*args, &block)
|
@@ -76,7 +77,13 @@ module Delayed
|
|
76
77
|
def execute(*args, &block)
|
77
78
|
@before.each { |c| c.call(*args) }
|
78
79
|
result = @around.call(*args, &block)
|
79
|
-
@after.each
|
80
|
+
@after.each do |c|
|
81
|
+
if c.arity == args.length + 1 # don't fail for methods that don't define `result:`
|
82
|
+
c.call(*args, result: result)
|
83
|
+
else
|
84
|
+
c.call(*args)
|
85
|
+
end
|
86
|
+
end
|
80
87
|
result
|
81
88
|
end
|
82
89
|
|
data/lib/delayed/log_tailer.rb
CHANGED
@@ -5,8 +5,8 @@ module Delayed
|
|
5
5
|
def run
|
6
6
|
if Rails.logger.respond_to?(:log_path)
|
7
7
|
log_path = Rails.logger.log_path
|
8
|
-
elsif Rails.logger.instance_variable_get(
|
9
|
-
log_path = Rails.logger.instance_variable_get(
|
8
|
+
elsif Rails.logger.instance_variable_get(:@logdev).try(:instance_variable_get, "@dev").try(:path)
|
9
|
+
log_path = Rails.logger.instance_variable_get(:@logdev).instance_variable_get(:@dev).path
|
10
10
|
else
|
11
11
|
return
|
12
12
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "debug_inspector" if
|
3
|
+
require "debug_inspector" if Rails.env.test? || Rails.env.development?
|
4
4
|
|
5
5
|
module Delayed
|
6
6
|
module MessageSending
|
@@ -19,7 +19,7 @@ module Delayed
|
|
19
19
|
@object.protected_methods.exclude?(method) &&
|
20
20
|
@object.private_methods.exclude?(method)
|
21
21
|
|
22
|
-
sender_is_object = @sender
|
22
|
+
sender_is_object = @sender.equal?(@object)
|
23
23
|
sender_is_class = @sender.is_a?(@object.class)
|
24
24
|
|
25
25
|
# even if the call is async, if the call is _going_ to generate an error, we make it synchronous
|
@@ -54,8 +54,10 @@ module Delayed
|
|
54
54
|
|
55
55
|
if ::Delayed::Job != ::Delayed::Backend::ActiveRecord::Job || connection != ::Delayed::Job.connection
|
56
56
|
connection.after_transaction_commit do
|
57
|
-
::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object,
|
58
|
-
|
57
|
+
::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object,
|
58
|
+
method,
|
59
|
+
args: args,
|
60
|
+
kwargs: kwargs,
|
59
61
|
on_failure: on_failure,
|
60
62
|
on_permanent_failure: on_permanent_failure,
|
61
63
|
sender: @sender),
|
@@ -65,7 +67,8 @@ module Delayed
|
|
65
67
|
end
|
66
68
|
end
|
67
69
|
|
68
|
-
result = ::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object,
|
70
|
+
result = ::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object,
|
71
|
+
method,
|
69
72
|
args: args,
|
70
73
|
kwargs: kwargs,
|
71
74
|
on_failure: on_failure,
|
@@ -1,7 +1,15 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module Delayed
|
4
|
-
PerformableMethod = Struct.new(:object,
|
4
|
+
PerformableMethod = Struct.new(:object,
|
5
|
+
:method, # rubocop:disable Lint/StructNewOverride
|
6
|
+
:args,
|
7
|
+
:kwargs,
|
8
|
+
:fail_cb,
|
9
|
+
:permanent_fail_cb,
|
10
|
+
:sender,
|
11
|
+
:sender_is_object,
|
12
|
+
:sender_is_class) do
|
5
13
|
def initialize(object, method, args: [], kwargs: {}, on_failure: nil, on_permanent_failure: nil, sender: nil)
|
6
14
|
raise NoMethodError, "undefined method `#{method}' for #{object.inspect}" unless object.respond_to?(method, true)
|
7
15
|
|
@@ -11,13 +19,10 @@ module Delayed
|
|
11
19
|
self.method = method.to_sym
|
12
20
|
self.fail_cb = on_failure
|
13
21
|
self.permanent_fail_cb = on_permanent_failure
|
14
|
-
self.
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
# if for some reason you can't dump the sender, just drop it
|
19
|
-
self.sender = nil
|
20
|
-
end
|
22
|
+
self.sender_is_object = sender.equal?(object)
|
23
|
+
self.sender_is_class = sender.is_a?(object.class)
|
24
|
+
# bypass visibility checks (see MessageSending#__calculate_sender_for_delay)
|
25
|
+
self.sender_is_object = true if sender.nil?
|
21
26
|
end
|
22
27
|
|
23
28
|
def display_name
|
@@ -32,14 +37,15 @@ module Delayed
|
|
32
37
|
def perform
|
33
38
|
kwargs = self.kwargs || {}
|
34
39
|
|
35
|
-
|
36
|
-
|
40
|
+
# back-compat for jobs queued before we assigned these in initialize
|
41
|
+
self.sender_is_object = sender.equal?(object) if sender_is_object.nil?
|
42
|
+
self.sender_is_class = sender.is_a?(object.class) if sender_is_class.nil?
|
37
43
|
|
38
|
-
if
|
44
|
+
if sender_is_object || (sender_is_class && object.protected_methods.include?(method))
|
39
45
|
if kwargs.empty?
|
40
|
-
object.
|
46
|
+
object.__send__(method, *args)
|
41
47
|
else
|
42
|
-
object.
|
48
|
+
object.__send__(method, *args, **kwargs)
|
43
49
|
end
|
44
50
|
elsif kwargs.empty?
|
45
51
|
object.public_send(method, *args)
|
@@ -59,9 +65,9 @@ module Delayed
|
|
59
65
|
def deep_de_ar_ize(arg)
|
60
66
|
case arg
|
61
67
|
when Hash
|
62
|
-
"{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(
|
68
|
+
"{#{arg.map { |k, v| "#{deep_de_ar_ize(k)} => #{deep_de_ar_ize(v)}" }.join(", ")}}"
|
63
69
|
when Array
|
64
|
-
"[#{arg.map { |a| deep_de_ar_ize(a) }.join(
|
70
|
+
"[#{arg.map { |a| deep_de_ar_ize(a) }.join(", ")}]"
|
65
71
|
when ActiveRecord::Base
|
66
72
|
"#{arg.class}.find(#{arg.id})"
|
67
73
|
else
|
@@ -73,7 +79,7 @@ module Delayed
|
|
73
79
|
obj_name = object.is_a?(ActiveRecord::Base) ? "#{object.class}.find(#{object.id}).#{method}" : display_name
|
74
80
|
kgs = kwargs || {}
|
75
81
|
kwargs_str = kgs.map { |(k, v)| ", #{k}: #{deep_de_ar_ize(v)}" }.join
|
76
|
-
"#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(
|
82
|
+
"#{obj_name}(#{args.map { |a| deep_de_ar_ize(a) }.join(", ")}#{kwargs_str})"
|
77
83
|
end
|
78
84
|
end
|
79
85
|
end
|
data/lib/delayed/periodic.rb
CHANGED
@@ -36,7 +36,7 @@ module Delayed
|
|
36
36
|
Delayed::Job.transaction do
|
37
37
|
# for db performance reasons, we only need one process doing this at a time
|
38
38
|
# so if we can't get an advisory lock, just abort. we'll try again soon
|
39
|
-
|
39
|
+
next unless Delayed::Job.attempt_advisory_lock("Delayed::Periodic#audit_queue")
|
40
40
|
|
41
41
|
perform_audit!
|
42
42
|
end
|
@@ -83,7 +83,7 @@ module Delayed
|
|
83
83
|
def tag
|
84
84
|
"periodic: #{@name}"
|
85
85
|
end
|
86
|
-
|
86
|
+
alias_method :display_name, :tag
|
87
87
|
|
88
88
|
def self.now
|
89
89
|
Time.zone.now
|
data/lib/delayed/pool.rb
CHANGED
@@ -67,11 +67,21 @@ module Delayed
|
|
67
67
|
|
68
68
|
unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
|
69
69
|
say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs.positive?
|
70
|
-
|
70
|
+
return if Rails.env.test?
|
71
|
+
|
72
|
+
if Rails.version < "6.1"
|
73
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!
|
74
|
+
else
|
75
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!(nil)
|
76
|
+
end
|
71
77
|
end
|
72
78
|
|
73
79
|
def spawn_all_workers
|
74
|
-
|
80
|
+
if Rails.version < "6.1"
|
81
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!
|
82
|
+
else
|
83
|
+
ActiveRecord::Base.connection_handler.clear_all_connections!(nil)
|
84
|
+
end
|
75
85
|
|
76
86
|
if @config[:work_queue] == "parent_process"
|
77
87
|
@work_queue = WorkQueue::ParentProcess.new
|
data/lib/delayed/server.rb
CHANGED
@@ -16,7 +16,7 @@ module Delayed
|
|
16
16
|
# Rails will take care of establishing the DB connection for us if there is
|
17
17
|
# an application present
|
18
18
|
if using_active_record? && !ActiveRecord::Base.connected?
|
19
|
-
ActiveRecord::Base.establish_connection(ENV
|
19
|
+
ActiveRecord::Base.establish_connection(ENV.fetch("DATABASE_URL", nil))
|
20
20
|
end
|
21
21
|
|
22
22
|
@allow_update = args.length.positive? && args[0][:update]
|
@@ -37,7 +37,13 @@ module Delayed
|
|
37
37
|
|
38
38
|
# Release any used connections back to the pool
|
39
39
|
after do
|
40
|
-
|
40
|
+
if using_active_record?
|
41
|
+
if Rails.version < "6.1"
|
42
|
+
::ActiveRecord::Base.clear_active_connections!
|
43
|
+
else
|
44
|
+
::ActiveRecord::Base.clear_active_connections!(nil)
|
45
|
+
end
|
46
|
+
end
|
41
47
|
end
|
42
48
|
|
43
49
|
configure :development do
|
data/lib/delayed/settings.rb
CHANGED
@@ -26,6 +26,7 @@ module Delayed
|
|
26
26
|
:sleep_delay,
|
27
27
|
:sleep_delay_stagger,
|
28
28
|
:slow_exit_timeout,
|
29
|
+
:stranded_run_at_grace_period,
|
29
30
|
:worker_health_check_type,
|
30
31
|
:worker_health_check_config,
|
31
32
|
:worker_procname_prefix
|
@@ -33,6 +34,7 @@ module Delayed
|
|
33
34
|
|
34
35
|
SETTINGS_WITH_ARGS = %i[
|
35
36
|
job_detailed_log_format
|
37
|
+
job_short_log_format
|
36
38
|
num_strands
|
37
39
|
].freeze
|
38
40
|
|
@@ -131,12 +133,14 @@ module Delayed
|
|
131
133
|
self.fetch_batch_size = 5
|
132
134
|
self.select_random_from_batch = false
|
133
135
|
self.silence_periodic_log = false
|
136
|
+
self.stranded_run_at_grace_period = 10
|
134
137
|
|
135
138
|
self.num_strands = ->(_strand_name) {}
|
136
139
|
self.default_job_options = -> { {} }
|
137
140
|
self.job_detailed_log_format = lambda { |job|
|
138
|
-
job.to_json(include_root: false, only: %w[tag strand priority attempts created_at max_attempts source])
|
141
|
+
job.to_json(include_root: false, only: %w[tag strand singleton priority attempts created_at max_attempts source])
|
139
142
|
}
|
143
|
+
self.job_short_log_format = ->(_job) { "" }
|
140
144
|
|
141
145
|
# Send workers KILL after QUIT if they haven't exited within the
|
142
146
|
# slow_exit_timeout
|
data/lib/delayed/version.rb
CHANGED
@@ -163,7 +163,7 @@ module Delayed
|
|
163
163
|
forced_latency: forced_latency
|
164
164
|
)
|
165
165
|
logger.debug(
|
166
|
-
"Fetched and locked #{response.values.flatten.size} new jobs for workers (#{response.keys.join(
|
166
|
+
"Fetched and locked #{response.values.flatten.size} new jobs for workers (#{response.keys.join(", ")})."
|
167
167
|
)
|
168
168
|
response.each do |(worker_name, locked_jobs)|
|
169
169
|
if worker_name == prefetch_owner
|
@@ -226,7 +226,11 @@ module Delayed
|
|
226
226
|
|
227
227
|
# otherwise just reconnect and let it retry
|
228
228
|
logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
|
229
|
-
|
229
|
+
if Rails.version < "6.1"
|
230
|
+
::Delayed::Job.clear_all_connections!
|
231
|
+
else
|
232
|
+
::Delayed::Job.clear_all_connections!(nil)
|
233
|
+
end
|
230
234
|
end
|
231
235
|
end
|
232
236
|
|
@@ -263,7 +267,7 @@ module Delayed
|
|
263
267
|
end
|
264
268
|
|
265
269
|
def prefetch_owner
|
266
|
-
"prefetch:#{Socket.gethostname rescue
|
270
|
+
"prefetch:#{Socket.gethostname rescue "X"}"
|
267
271
|
end
|
268
272
|
|
269
273
|
def parent_exited?
|
@@ -34,7 +34,7 @@ module Delayed
|
|
34
34
|
# no other worker is trying to do this right now (and if we abandon the
|
35
35
|
# operation, the transaction will end, releasing the advisory lock).
|
36
36
|
result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
|
37
|
-
|
37
|
+
next unless result
|
38
38
|
|
39
39
|
horizon = 5.minutes.ago
|
40
40
|
|