inst-jobs 0.16.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/db/migrate/20101216224513_create_delayed_jobs.rb +2 -0
- data/db/migrate/20110208031356_add_delayed_jobs_tag.rb +2 -0
- data/db/migrate/20110426161613_add_delayed_jobs_max_attempts.rb +2 -0
- data/db/migrate/20110516225834_add_delayed_jobs_strand.rb +2 -0
- data/db/migrate/20110531144916_cleanup_delayed_jobs_indexes.rb +2 -0
- data/db/migrate/20110610213249_optimize_delayed_jobs.rb +2 -0
- data/db/migrate/20110831210257_add_delayed_jobs_next_in_strand.rb +2 -0
- data/db/migrate/20120510004759_delayed_jobs_delete_trigger_lock_for_update.rb +2 -0
- data/db/migrate/20120531150712_drop_psql_jobs_pop_fn.rb +2 -0
- data/db/migrate/20120607164022_delayed_jobs_use_advisory_locks.rb +2 -0
- data/db/migrate/20120607181141_index_jobs_on_locked_by.rb +2 -0
- data/db/migrate/20120608191051_add_jobs_run_at_index.rb +2 -0
- data/db/migrate/20120927184213_change_delayed_jobs_handler_to_text.rb +2 -0
- data/db/migrate/20140505215131_add_failed_jobs_original_job_id.rb +2 -0
- data/db/migrate/20140505215510_copy_failed_jobs_original_id.rb +2 -0
- data/db/migrate/20140505223637_drop_failed_jobs_original_id.rb +2 -0
- data/db/migrate/20140512213941_add_source_to_jobs.rb +2 -0
- data/db/migrate/20150807133223_add_max_concurrent_to_jobs.rb +2 -0
- data/db/migrate/20151123210429_add_expires_at_to_jobs.rb +2 -0
- data/db/migrate/20151210162949_improve_max_concurrent.rb +2 -0
- data/db/migrate/20161206323555_add_back_default_string_limits_jobs.rb +2 -0
- data/db/migrate/20181217155351_speed_up_max_concurrent_triggers.rb +2 -0
- data/db/migrate/20190726154743_make_critical_columns_not_null.rb +2 -0
- data/db/migrate/20200330230722_add_id_to_get_delayed_jobs_index.rb +2 -0
- data/db/migrate/20200824222232_speed_up_max_concurrent_delete_trigger.rb +2 -0
- data/db/migrate/20200825011002_add_strand_order_override.rb +2 -0
- data/lib/delayed/backend/active_record.rb +6 -4
- data/lib/delayed/backend/base.rb +34 -20
- data/lib/delayed/backend/redis/functions.rb +2 -0
- data/lib/delayed/backend/redis/job.rb +2 -0
- data/lib/delayed/batch.rb +5 -3
- data/lib/delayed/cli.rb +2 -0
- data/lib/delayed/daemon.rb +2 -0
- data/lib/delayed/engine.rb +2 -0
- data/lib/delayed/job_tracking.rb +2 -0
- data/lib/delayed/lifecycle.rb +2 -0
- data/lib/delayed/log_tailer.rb +2 -0
- data/lib/delayed/logging.rb +2 -0
- data/lib/delayed/message_sending.rb +48 -111
- data/lib/delayed/performable_method.rb +15 -6
- data/lib/delayed/periodic.rb +6 -4
- data/lib/delayed/plugin.rb +2 -0
- data/lib/delayed/pool.rb +2 -0
- data/lib/delayed/server.rb +2 -0
- data/lib/delayed/server/helpers.rb +2 -0
- data/lib/delayed/settings.rb +2 -0
- data/lib/delayed/testing.rb +2 -0
- data/lib/delayed/version.rb +3 -1
- data/lib/delayed/work_queue/in_process.rb +2 -0
- data/lib/delayed/work_queue/parent_process.rb +2 -0
- data/lib/delayed/work_queue/parent_process/client.rb +2 -0
- data/lib/delayed/work_queue/parent_process/server.rb +2 -0
- data/lib/delayed/worker.rb +2 -0
- data/lib/delayed/worker/consul_health_check.rb +2 -0
- data/lib/delayed/worker/health_check.rb +2 -0
- data/lib/delayed/worker/null_health_check.rb +2 -0
- data/lib/delayed/worker/process_helper.rb +2 -0
- data/lib/delayed/yaml_extensions.rb +2 -0
- data/lib/delayed_job.rb +2 -0
- data/lib/inst-jobs.rb +2 -0
- data/spec/active_record_job_spec.rb +4 -2
- data/spec/delayed/cli_spec.rb +2 -0
- data/spec/delayed/daemon_spec.rb +2 -0
- data/spec/delayed/server_spec.rb +2 -0
- data/spec/delayed/settings_spec.rb +2 -0
- data/spec/delayed/work_queue/in_process_spec.rb +2 -0
- data/spec/delayed/work_queue/parent_process/client_spec.rb +2 -0
- data/spec/delayed/work_queue/parent_process/server_spec.rb +2 -0
- data/spec/delayed/work_queue/parent_process_spec.rb +2 -0
- data/spec/delayed/worker/consul_health_check_spec.rb +2 -0
- data/spec/delayed/worker/health_check_spec.rb +2 -0
- data/spec/delayed/worker_spec.rb +2 -0
- data/spec/gemfiles/42.gemfile.lock +192 -0
- data/spec/gemfiles/50.gemfile.lock +197 -0
- data/spec/gemfiles/51.gemfile.lock +198 -0
- data/spec/gemfiles/52.gemfile.lock +206 -0
- data/spec/gemfiles/60.gemfile.lock +222 -0
- data/spec/migrate/20140924140513_add_story_table.rb +2 -0
- data/spec/redis_job_spec.rb +10 -8
- data/spec/sample_jobs.rb +2 -0
- data/spec/shared/delayed_batch.rb +17 -15
- data/spec/shared/delayed_method.rb +49 -204
- data/spec/shared/performable_method.rb +11 -9
- data/spec/shared/shared_backend.rb +27 -25
- data/spec/shared/testing.rb +7 -5
- data/spec/shared/worker.rb +15 -13
- data/spec/shared_jobs_specs.rb +2 -0
- data/spec/spec_helper.rb +3 -1
- metadata +17 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8c754ed71778d5b3f08641e133be947fae204a56dcee71aef08a7780a7d6afa5
|
4
|
+
data.tar.gz: 10949318e78e8e9d56b92ee7f2d14bfd647da20ff70578d894e4d1e85441d2fe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a014e191bc0d6ccd083340b6d5c4bca0b7e03439aca459f79a5a704d93d16c42dd9e4855ca5985f7b1e4fb1fe62eead160cb6cb356c73b00ac297dc30112d8a
|
7
|
+
data.tar.gz: 53997c66017cf6db561104054379cb9e7e934fcc56c54d552b7629ff29eeb58de37c8674088661ba683e7e4d8b19234f26021819753ac06a6c738dd435c3f169
|
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
class ActiveRecord::Base
|
2
4
|
def self.load_for_delayed_job(id)
|
3
5
|
if id
|
@@ -105,7 +107,7 @@ module Delayed
|
|
105
107
|
# so rather than changing the strand and balancing at queue time,
|
106
108
|
# this keeps the strand intact and uses triggers to limit the number running
|
107
109
|
def self.n_strand_options(strand_name, num_strands)
|
108
|
-
{:
|
110
|
+
{ strand: strand_name, max_concurrent: num_strands }
|
109
111
|
end
|
110
112
|
|
111
113
|
def self.current
|
@@ -125,8 +127,8 @@ module Delayed
|
|
125
127
|
end
|
126
128
|
|
127
129
|
# a nice stress test:
|
128
|
-
# 10_000.times { |i| Kernel.
|
129
|
-
# 500.times { |i| "ohai".
|
130
|
+
# 10_000.times { |i| Kernel.delay(strand: 's1', run_at: (24.hours.ago + (rand(24.hours.to_i))).system("echo #{i} >> test1.txt") }
|
131
|
+
# 500.times { |i| "ohai".delay(run_at: (12.hours.ago + (rand(24.hours.to_i))).reverse }
|
130
132
|
# then fire up your workers
|
131
133
|
# you can check out strand correctness: diff test1.txt <(sort -n test1.txt)
|
132
134
|
def self.ready_to_run(forced_latency: nil)
|
@@ -288,7 +290,7 @@ module Delayed
|
|
288
290
|
lock(lock)
|
289
291
|
jobs_with_row_number = all.from(target_jobs).
|
290
292
|
select("id, ROW_NUMBER() OVER () AS row_number")
|
291
|
-
updates = "locked_by = CASE row_number "
|
293
|
+
updates = +"locked_by = CASE row_number "
|
292
294
|
effective_worker_names.each_with_index do |worker, i|
|
293
295
|
updates << "WHEN #{i + 1} THEN #{connection.quote(worker)} "
|
294
296
|
end
|
data/lib/delayed/backend/base.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Delayed
|
2
4
|
module Backend
|
3
5
|
class DeserializationError < StandardError
|
@@ -28,19 +30,31 @@ module Delayed
|
|
28
30
|
# The first argument should be an object that respond_to?(:perform)
|
29
31
|
# The rest should be named arguments, these keys are expected:
|
30
32
|
# :priority, :run_at, :queue, :strand, :singleton
|
31
|
-
# Example: Delayed::Job.enqueue(object, :
|
32
|
-
def enqueue(
|
33
|
-
|
33
|
+
# Example: Delayed::Job.enqueue(object, priority: 0, run_at: time, queue: queue)
|
34
|
+
def enqueue(object,
|
35
|
+
priority: default_priority,
|
36
|
+
run_at: nil,
|
37
|
+
expires_at: nil,
|
38
|
+
queue: Delayed::Settings.queue,
|
39
|
+
strand: nil,
|
40
|
+
singleton: nil,
|
41
|
+
n_strand: nil,
|
42
|
+
max_attempts: Delayed::Settings.max_attempts,
|
43
|
+
**kwargs)
|
44
|
+
|
34
45
|
unless object.respond_to?(:perform)
|
35
46
|
raise ArgumentError, 'Cannot enqueue items which do not respond to perform'
|
36
47
|
end
|
37
48
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
49
|
+
kwargs = Settings.default_job_options.merge(kwargs)
|
50
|
+
kwargs[:payload_object] = object
|
51
|
+
kwargs[:priority] = priority
|
52
|
+
kwargs[:run_at] = run_at if run_at
|
53
|
+
kwargs[:strand] = strand
|
54
|
+
kwargs[:max_attempts] = max_attempts
|
55
|
+
kwargs[:source] = Marginalia::Comment.construct_comment if defined?(Marginalia) && Marginalia::Comment.components
|
56
|
+
kwargs[:expires_at] = expires_at
|
57
|
+
kwargs[:queue] = queue
|
44
58
|
|
45
59
|
# If two parameters are given to n_strand, the first param is used
|
46
60
|
# as the strand name for looking up the Setting, while the second
|
@@ -49,8 +63,8 @@ module Delayed
|
|
49
63
|
# For instance, you can pass ["my_job_type", # root_account.global_id]
|
50
64
|
# to get a set of n strands per root account, and you can apply the
|
51
65
|
# same default to all.
|
52
|
-
if
|
53
|
-
strand_name, ext =
|
66
|
+
if n_strand
|
67
|
+
strand_name, ext = n_strand
|
54
68
|
|
55
69
|
if ext
|
56
70
|
full_strand_name = "#{strand_name}/#{ext}"
|
@@ -62,18 +76,18 @@ module Delayed
|
|
62
76
|
num_strands ||= Delayed::Settings.num_strands.call(strand_name)
|
63
77
|
num_strands = num_strands ? num_strands.to_i : 1
|
64
78
|
|
65
|
-
|
79
|
+
kwargs.merge!(n_strand_options(full_strand_name, num_strands))
|
66
80
|
end
|
67
81
|
|
68
|
-
if
|
69
|
-
|
70
|
-
job = self.create_singleton(
|
71
|
-
elsif batches &&
|
72
|
-
batch_enqueue_args =
|
73
|
-
batches[batch_enqueue_args] <<
|
82
|
+
if singleton
|
83
|
+
kwargs[:strand] = singleton
|
84
|
+
job = self.create_singleton(**kwargs)
|
85
|
+
elsif batches && strand.nil? && run_at.nil?
|
86
|
+
batch_enqueue_args = kwargs.slice(*self.batch_enqueue_args)
|
87
|
+
batches[batch_enqueue_args] << kwargs
|
74
88
|
return true
|
75
89
|
else
|
76
|
-
job = self.create(
|
90
|
+
job = self.create(**kwargs)
|
77
91
|
end
|
78
92
|
|
79
93
|
JobTracking.job_created(job)
|
@@ -87,7 +101,7 @@ module Delayed
|
|
87
101
|
def n_strand_options(strand_name, num_strands)
|
88
102
|
strand_num = num_strands > 1 ? rand(num_strands) + 1 : 1
|
89
103
|
strand_name += ":#{strand_num}" if strand_num > 1
|
90
|
-
{:
|
104
|
+
{ strand: strand_name }
|
91
105
|
end
|
92
106
|
|
93
107
|
def in_delayed_job?
|
data/lib/delayed/batch.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Delayed
|
2
4
|
module Batch
|
3
5
|
class PerformableBatch < Struct.new(:mode, :items)
|
@@ -18,7 +20,7 @@ module Delayed
|
|
18
20
|
end
|
19
21
|
|
20
22
|
def jobs
|
21
|
-
items.map { |opts| Delayed::Job.new(opts) }
|
23
|
+
items.map { |opts| Delayed::Job.new(**opts) }
|
22
24
|
end
|
23
25
|
end
|
24
26
|
|
@@ -45,9 +47,9 @@ module Delayed
|
|
45
47
|
elsif batch.size == 1
|
46
48
|
args = batch.first.merge(batch_args)
|
47
49
|
payload_object = args.delete(:payload_object)
|
48
|
-
Delayed::Job.enqueue(payload_object, args)
|
50
|
+
Delayed::Job.enqueue(payload_object, **args)
|
49
51
|
else
|
50
|
-
Delayed::Job.enqueue(Delayed::Batch::PerformableBatch.new(mode, batch), enqueue_args.merge(batch_args))
|
52
|
+
Delayed::Job.enqueue(Delayed::Batch::PerformableBatch.new(mode, batch), **enqueue_args.merge(batch_args))
|
51
53
|
end
|
52
54
|
end
|
53
55
|
end
|
data/lib/delayed/cli.rb
CHANGED
data/lib/delayed/daemon.rb
CHANGED
data/lib/delayed/engine.rb
CHANGED
data/lib/delayed/job_tracking.rb
CHANGED
data/lib/delayed/lifecycle.rb
CHANGED
data/lib/delayed/log_tailer.rb
CHANGED
data/lib/delayed/logging.rb
CHANGED
@@ -1,109 +1,67 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Delayed
|
2
4
|
module MessageSending
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
def send_later_enqueue_args(method, enqueue_args = {}, *args)
|
8
|
-
enqueue_args = enqueue_args.dup
|
9
|
-
# support procs/methods as enqueue arguments
|
10
|
-
enqueue_args.each do |k,v|
|
11
|
-
if v.respond_to?(:call)
|
12
|
-
enqueue_args[k] = v.call(self)
|
13
|
-
end
|
5
|
+
class DelayProxy < BasicObject
|
6
|
+
def initialize(object, enqueue_args)
|
7
|
+
@object = object
|
8
|
+
@enqueue_args = enqueue_args
|
14
9
|
end
|
15
10
|
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
connection.
|
28
|
-
|
29
|
-
|
11
|
+
def method_missing(method, *args, **kwargs)
|
12
|
+
ignore_transaction = @enqueue_args.delete(:ignore_transaction)
|
13
|
+
on_failure = @enqueue_args.delete(:on_failure)
|
14
|
+
on_permanent_failure = @enqueue_args.delete(:on_permanent_failure)
|
15
|
+
if !ignore_transaction
|
16
|
+
# delay queuing up the job in another database until the results of the current
|
17
|
+
# transaction are visible
|
18
|
+
connection = @object.class.connection if @object.class.respond_to?(:connection)
|
19
|
+
connection ||= @object.connection if @object.respond_to?(:connection)
|
20
|
+
connection ||= ::ActiveRecord::Base.connection
|
21
|
+
|
22
|
+
if (::Delayed::Job != ::Delayed::Backend::ActiveRecord::Job || connection != ::Delayed::Job.connection)
|
23
|
+
connection.after_transaction_commit do
|
24
|
+
::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object, method,
|
25
|
+
args: args, kwargs: kwargs,
|
26
|
+
on_failure: on_failure,
|
27
|
+
on_permanent_failure: on_permanent_failure),
|
28
|
+
**@enqueue_args)
|
29
|
+
end
|
30
|
+
return nil
|
30
31
|
end
|
31
|
-
return nil
|
32
32
|
end
|
33
|
-
end
|
34
|
-
|
35
|
-
result = Delayed::Job.enqueue(Delayed::PerformableMethod.new(self, method.to_sym, args,
|
36
|
-
on_failure, on_permanent_failure), enqueue_args)
|
37
|
-
result = nil unless no_delay
|
38
|
-
result
|
39
|
-
end
|
40
|
-
|
41
|
-
def send_later_with_queue(method, queue, *args)
|
42
|
-
send_later_enqueue_args(method, { :queue => queue }, *args)
|
43
|
-
end
|
44
|
-
|
45
|
-
def send_at(time, method, *args)
|
46
|
-
send_later_enqueue_args(method,
|
47
|
-
{ :run_at => time }, *args)
|
48
|
-
end
|
49
|
-
|
50
|
-
def send_at_with_queue(time, method, queue, *args)
|
51
|
-
send_later_enqueue_args(method,
|
52
|
-
{ :run_at => time, :queue => queue },
|
53
|
-
*args)
|
54
|
-
end
|
55
|
-
|
56
|
-
def send_later_unless_in_job(method, *args)
|
57
|
-
if Delayed::Job.in_delayed_job?
|
58
|
-
send(method, *args)
|
59
|
-
else
|
60
|
-
send_later(method, *args)
|
61
|
-
end
|
62
|
-
nil # can't rely on the type of return value, so return nothing
|
63
|
-
end
|
64
33
|
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
34
|
+
result = ::Delayed::Job.enqueue(::Delayed::PerformableMethod.new(@object, method,
|
35
|
+
args: args,
|
36
|
+
kwargs: kwargs,
|
37
|
+
on_failure: on_failure,
|
38
|
+
on_permanent_failure: on_permanent_failure),
|
39
|
+
**@enqueue_args)
|
40
|
+
result = nil unless ignore_transaction
|
41
|
+
result
|
70
42
|
end
|
71
43
|
end
|
72
44
|
|
73
|
-
def
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
end
|
80
|
-
|
81
|
-
def send_now_or_later(_when, *args)
|
82
|
-
if _when == :now
|
83
|
-
send(*args)
|
84
|
-
else
|
85
|
-
send_later(*args)
|
45
|
+
def delay(**enqueue_args)
|
46
|
+
# support procs/methods as enqueue arguments
|
47
|
+
enqueue_args.each do |k,v|
|
48
|
+
if v.respond_to?(:call)
|
49
|
+
enqueue_args[k] = v.call(self)
|
50
|
+
end
|
86
51
|
end
|
87
|
-
|
88
|
-
|
89
|
-
def send_now_or_later_if_production(_when, *args)
|
90
|
-
if _when == :now
|
91
|
-
send(*args)
|
92
|
-
else
|
93
|
-
send_later_if_production(*args)
|
52
|
+
if enqueue_args.delete(:synchronous)
|
53
|
+
return self
|
94
54
|
end
|
55
|
+
DelayProxy.new(self, enqueue_args)
|
95
56
|
end
|
96
57
|
|
97
58
|
module ClassMethods
|
98
59
|
KWARG_ARG_TYPES = %i{key keyreq keyrest}.freeze
|
99
60
|
private_constant :KWARG_ARG_TYPES
|
100
61
|
|
101
|
-
def
|
62
|
+
def handle_asynchronously(method_name, **enqueue_args)
|
102
63
|
aliased_method, punctuation = method_name.to_s.sub(/([?!=])$/, ''), $1
|
103
64
|
|
104
|
-
# we still need this for backwards compatibility
|
105
|
-
without_method = "#{aliased_method}_without_send_later#{punctuation}"
|
106
|
-
|
107
65
|
if public_method_defined?(method_name)
|
108
66
|
visibility = :public
|
109
67
|
elsif private_method_defined?(method_name)
|
@@ -114,49 +72,28 @@ module Delayed
|
|
114
72
|
|
115
73
|
if has_kwargs? method_name
|
116
74
|
generated_delayed_methods.class_eval do
|
117
|
-
define_method
|
118
|
-
send(method_name, *args, synchronous: true, **kwargs)
|
119
|
-
end
|
120
|
-
|
121
|
-
define_method(method_name, -> (*args, synchronous: !default_async, **kwargs) do
|
75
|
+
define_method(method_name, -> (*args, synchronous: false, **kwargs) do
|
122
76
|
if synchronous
|
123
77
|
super(*args, **kwargs)
|
124
78
|
else
|
125
|
-
|
79
|
+
delay(**enqueue_args).method_missing(method_name, *args, synchronous: true, **kwargs)
|
126
80
|
end
|
127
81
|
end)
|
128
82
|
end
|
129
83
|
else
|
130
84
|
generated_delayed_methods.class_eval do
|
131
|
-
define_method
|
132
|
-
send(method_name, *args, synchronous: true)
|
133
|
-
end
|
134
|
-
|
135
|
-
define_method(method_name, -> (*args, synchronous: !default_async) do
|
85
|
+
define_method(method_name, -> (*args, synchronous: false) do
|
136
86
|
if synchronous
|
137
87
|
super(*args)
|
138
88
|
else
|
139
|
-
|
89
|
+
delay(**enqueue_args).method_missing(method_name, *args, synchronous: true)
|
140
90
|
end
|
141
91
|
end)
|
142
92
|
end
|
143
93
|
end
|
144
|
-
generated_delayed_methods.send(visibility, without_method)
|
145
94
|
generated_delayed_methods.send(visibility, method_name)
|
146
95
|
end
|
147
96
|
|
148
|
-
def handle_asynchronously(method, enqueue_args={})
|
149
|
-
add_send_later_methods(method, enqueue_args, true)
|
150
|
-
end
|
151
|
-
|
152
|
-
def handle_asynchronously_with_queue(method, queue)
|
153
|
-
add_send_later_methods(method, {:queue => queue}, true)
|
154
|
-
end
|
155
|
-
|
156
|
-
def handle_asynchronously_if_production(method, enqueue_args={})
|
157
|
-
add_send_later_methods(method, enqueue_args, Rails.env.production?)
|
158
|
-
end
|
159
|
-
|
160
97
|
private
|
161
98
|
|
162
99
|
def generated_delayed_methods
|