delayed 2.1.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/app/models/delayed/job.rb +25 -18
  4. data/db/migrate/08_add_run_at_and_name_not_null_check.rb +34 -0
  5. data/db/migrate/09_validate_run_at_and_name_not_null.rb +41 -0
  6. data/lib/delayed/active_job_adapter.rb +41 -8
  7. data/lib/delayed/backend/base.rb +67 -11
  8. data/lib/delayed/backend/job_preparer.rb +33 -0
  9. data/lib/delayed/exceptions.rb +2 -0
  10. data/lib/delayed/lifecycle.rb +1 -1
  11. data/lib/delayed/monitor.rb +69 -43
  12. data/lib/delayed/plugins/instrumentation.rb +9 -3
  13. data/lib/delayed/version.rb +1 -1
  14. data/lib/delayed/worker.rb +2 -8
  15. data/spec/delayed/__snapshots__/monitor_spec.rb.snap +447 -1170
  16. data/spec/delayed/active_job_adapter_spec.rb +285 -46
  17. data/spec/delayed/job_spec.rb +218 -32
  18. data/spec/delayed/monitor_spec.rb +18 -19
  19. data/spec/delayed/plugins/instrumentation_spec.rb +41 -0
  20. data/spec/helper.rb +18 -6
  21. data/spec/lifecycle_spec.rb +1 -1
  22. data/spec/message_sending_spec.rb +3 -13
  23. data/spec/performable_method_spec.rb +0 -6
  24. data/spec/sample_jobs.rb +0 -10
  25. metadata +12 -10
  26. /data/db/migrate/{1_create_delayed_jobs.rb → 01_create_delayed_jobs.rb} +0 -0
  27. /data/db/migrate/{2_add_name_to_delayed_jobs.rb → 02_add_name_to_delayed_jobs.rb} +0 -0
  28. /data/db/migrate/{3_add_index_to_delayed_jobs_name.rb → 03_add_index_to_delayed_jobs_name.rb} +0 -0
  29. /data/db/migrate/{4_index_live_jobs.rb → 04_index_live_jobs.rb} +0 -0
  30. /data/db/migrate/{5_index_failed_jobs.rb → 05_index_failed_jobs.rb} +0 -0
  31. /data/db/migrate/{6_set_postgres_fillfactor.rb → 06_set_postgres_fillfactor.rb} +0 -0
  32. /data/db/migrate/{7_remove_legacy_index.rb → 07_remove_legacy_index.rb} +0 -0
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 984a910472fc4f1dedafcf4060cf076bcda408196bfa80f08039ed2527533a41
4
- data.tar.gz: 98f7de42e2964ee1576ba6ae3fc2b39cbe76a841263054a5b9634f7e55ce1114
3
+ metadata.gz: 5fac8946441b682c8df0e1365cd446cb7d507e3831f8155d692abe6ef1d52003
4
+ data.tar.gz: 602f57430c6ac8b7c94bd248f270ffe768d65de07d50bc2e482bfed67fc1ca7c
5
5
  SHA512:
6
- metadata.gz: dcb018c429ff591409796e410c570f49408eb0b8363759f6c8970ac4ee1773abccc117b5398c3596ecf8491c0e70db548bd4a2ea54ef39a77fa78da43b6cbfe4
7
- data.tar.gz: 020a785c7bfc81456b8cc7b8ddace55fad1f9468edb3b9cef0ef066eb6aed64b580d30974b48ce97bc7b1602294c9901c7c26fa294d3e4b4ecebd177aef677c8
6
+ metadata.gz: 3d5a2383bd876172d81e9d0926d818afe6f989af71e4143c970daf0ea5149f63cad5ccf0029be97e59765b4ceae77dc555578a76fb955ed67f1caaaf92c71352
7
+ data.tar.gz: 0f33ed115647bbacfdfe27e9db20e075c2b7374d7ee23e57bbe1a983dacfc7d1cbea6e295fb7d672203664f98d151035675e4fccb35eeb2b627dadab921f2a4d
data/README.md CHANGED
@@ -284,11 +284,11 @@ The following events will be emitted automatically by workers as jobs are reserv
284
284
  - **delayed.job.run** - an event measuring the duration of a job's execution
285
285
  - **delayed.job.error** - an event indicating that a job has errored and may be retried (no duration attached)
286
286
  - **delayed.job.failure** - an event indicating that a job has permanently failed (no duration attached)
287
- - **delayed.job.enqueue** - an event measuring the time it takes to enqueue a job
287
+ - **delayed.job.enqueue** - an event measuring the time it takes to enqueue one or more jobs (fires once per `Delayed::Job.enqueue` call and once per `perform_all_later` / `enqueue_all` batch)
288
288
  - **delayed.worker.reserve_jobs** - an event measuring the duration of the job "pickup query"
289
289
 
290
- The "run", "error", "failure" and "enqueue" events will include a `:job` argument in the event's payload,
291
- providing access to the job instance.
290
+ The "run", "error", and "failure" events will include a `:job` argument in the event's payload,
291
+ providing access to the job instance. The "enqueue" event will include a `:jobs` array.
292
292
 
293
293
  ```ruby
294
294
  ActiveSupport::Notifications.subscribe('delayed.job.run') do |*args|
@@ -10,26 +10,22 @@ module Delayed
10
10
 
11
11
  # high-level queue states (live => erroring => failed)
12
12
  scope :live, -> { where(failed_at: nil) }
13
- scope :erroring, -> { where(arel_table[:attempts].gt(0)).merge(unscoped.live) }
13
+ scope :erroring, -> { where(erroring_clause).merge(unscoped.live) }
14
14
  scope :failed, -> { where.not(failed_at: nil) }
15
15
 
16
16
  # live queue states (future vs pending)
17
- scope :future, ->(as_of = db_time_now) { merge(unscoped.live).where(arel_table[:run_at].gt(as_of)) }
18
- scope :pending, ->(as_of = db_time_now) { merge(unscoped.live).where(arel_table[:run_at].lteq(as_of)) }
17
+ scope :future, ->(as_of = db_time_now) { merge(unscoped.live).where(future_clause(as_of)) }
18
+ scope :pending, ->(as_of = db_time_now) { merge(unscoped.live).where(pending_clause(as_of)) }
19
19
 
20
20
  # pending queue states (claimed vs claimable)
21
21
  scope :claimed, ->(as_of = db_time_now) {
22
- where(arel_table[:locked_at].gteq(db_time_now - lock_timeout))
23
- .merge(unscoped.pending(as_of))
22
+ where(claimed_clause(as_of)).merge(unscoped.pending(as_of))
24
23
  }
25
24
  scope :claimed_by, ->(worker, as_of = db_time_now) {
26
- where(locked_by: worker.name)
27
- .claimed(as_of)
25
+ where(locked_by: worker.name).claimed(as_of)
28
26
  }
29
27
  scope :claimable, ->(as_of = db_time_now) {
30
- where(locked_at: nil)
31
- .or(where(arel_table[:locked_at].lt(db_time_now - lock_timeout)))
32
- .merge(unscoped.pending(as_of))
28
+ where(claimable_clause(as_of)).merge(unscoped.pending(as_of))
33
29
  }
34
30
  scope :claimable_by, ->(worker, as_of = db_time_now) {
35
31
  claimable(as_of)
@@ -40,7 +36,25 @@ module Delayed
40
36
  .by_priority
41
37
  }
42
38
 
43
- before_save :set_default_run_at, :set_name
39
+ def self.erroring_clause
40
+ arel_table[:attempts].gt(0)
41
+ end
42
+
43
+ def self.future_clause(as_of = db_time_now)
44
+ arel_table[:run_at].gt(as_of)
45
+ end
46
+
47
+ def self.pending_clause(as_of = db_time_now)
48
+ arel_table[:run_at].lteq(as_of)
49
+ end
50
+
51
+ def self.claimed_clause(as_of = db_time_now)
52
+ arel_table[:locked_at].gteq(as_of - lock_timeout)
53
+ end
54
+
55
+ def self.claimable_clause(as_of = db_time_now)
56
+ arel_table[:locked_at].eq(nil).or arel_table[:locked_at].lt(as_of - lock_timeout)
57
+ end
44
58
 
45
59
  REENQUEUE_BUFFER = 30.seconds
46
60
 
@@ -265,12 +279,5 @@ module Delayed
265
279
  def attempts_alert?
266
280
  alert_attempts&.<= attempts
267
281
  end
268
-
269
- private
270
-
271
- def set_name
272
- # [feat:NameColumn] remove 'if' statement and use `||=` once the 'name' column is required.
273
- self.name = display_name if respond_to?(:name=) && attributes.fetch('name').nil?
274
- end
275
282
  end
276
283
  end
@@ -0,0 +1,34 @@
1
+ # frozen_string_literal: true
2
+
3
+ class AddRunAtAndNameNotNullCheck < ActiveRecord::Migration[6.0]
4
+ CONSTRAINTS = {
5
+ run_at: 'chk_delayed_jobs_run_at_not_null',
6
+ name: 'chk_delayed_jobs_name_not_null',
7
+ }.freeze
8
+
9
+ def up
10
+ return unless postgres?
11
+
12
+ CONSTRAINTS.each do |column, name|
13
+ execute <<~SQL.squish
14
+ ALTER TABLE delayed_jobs
15
+ ADD CONSTRAINT #{name}
16
+ CHECK (#{column} IS NOT NULL) NOT VALID
17
+ SQL
18
+ end
19
+ end
20
+
21
+ def down
22
+ return unless postgres?
23
+
24
+ CONSTRAINTS.each_value do |name|
25
+ execute "ALTER TABLE delayed_jobs DROP CONSTRAINT IF EXISTS #{name}"
26
+ end
27
+ end
28
+
29
+ private
30
+
31
+ def postgres?
32
+ connection.adapter_name == 'PostgreSQL'
33
+ end
34
+ end
@@ -0,0 +1,41 @@
1
+ # frozen_string_literal: true
2
+
3
+ class ValidateRunAtAndNameNotNull < ActiveRecord::Migration[6.0]
4
+ CONSTRAINTS = {
5
+ run_at: 'chk_delayed_jobs_run_at_not_null',
6
+ name: 'chk_delayed_jobs_name_not_null',
7
+ }.freeze
8
+
9
+ def up
10
+ CONSTRAINTS.each do |column, name|
11
+ if postgres?
12
+ execute "ALTER TABLE delayed_jobs VALIDATE CONSTRAINT #{name}"
13
+ change_column_null :delayed_jobs, column, false
14
+ execute "ALTER TABLE delayed_jobs DROP CONSTRAINT #{name}"
15
+ else
16
+ change_column_null :delayed_jobs, column, false
17
+ end
18
+ end
19
+ end
20
+
21
+ def down
22
+ CONSTRAINTS.each do |column, name|
23
+ if postgres?
24
+ change_column_null :delayed_jobs, column, true
25
+ execute <<~SQL.squish
26
+ ALTER TABLE delayed_jobs
27
+ ADD CONSTRAINT #{name}
28
+ CHECK (#{column} IS NOT NULL) NOT VALID
29
+ SQL
30
+ else
31
+ change_column_null :delayed_jobs, column, true
32
+ end
33
+ end
34
+ end
35
+
36
+ private
37
+
38
+ def postgres?
39
+ connection.adapter_name == 'PostgreSQL'
40
+ end
41
+ end
@@ -11,24 +11,57 @@ module Delayed
11
11
  end
12
12
 
13
13
  def enqueue_at(job, timestamp)
14
- _enqueue(job, run_at: Time.at(timestamp)) # rubocop:disable Rails/TimeZone
14
+ job.scheduled_at = Time.at(timestamp) # rubocop:disable Rails/TimeZone
15
+ _enqueue(job)
16
+ end
17
+
18
+ def enqueue_all(jobs)
19
+ return 0 if jobs.empty?
20
+
21
+ assert_jobs_safe_to_enqueue!(jobs)
22
+
23
+ delayed_jobs = jobs.map { |job| build_delayed_job(job) }
24
+ Delayed::Job.enqueue_all(delayed_jobs)
25
+
26
+ perform_post_enqueue_assignments(jobs, delayed_jobs)
27
+
28
+ jobs.size
15
29
  end
16
30
 
17
31
  private
18
32
 
19
- def _enqueue(job, opts = {})
20
- if enqueue_after_transaction_commit_enabled?(job)
21
- raise UnsafeEnqueueError, "The ':delayed' ActiveJob adapter is not compatible with enqueue_after_transaction_commit"
33
+ def _enqueue(job)
34
+ job.tap { |j| enqueue_all([j]) }
35
+ end
36
+
37
+ def assert_jobs_safe_to_enqueue!(jobs)
38
+ jobs.each do |job|
39
+ if enqueue_after_transaction_commit_enabled?(job)
40
+ raise UnsafeEnqueueError, "The ':delayed' ActiveJob adapter is not compatible with enqueue_after_transaction_commit"
41
+ end
22
42
  end
43
+ end
23
44
 
24
- opts.merge!({ queue: job.queue_name, priority: job.priority }.compact)
25
- .merge!(job.provider_attributes || {})
45
+ def build_delayed_job(job)
46
+ opts = { queue: job.queue_name, priority: job.priority }.compact
47
+ opts.merge!(job.provider_attributes || {})
48
+ opts[:run_at] = coerce_scheduled_at(job.scheduled_at) if job.scheduled_at
26
49
 
27
- Delayed::Job.enqueue(JobWrapper.new(job), opts).tap do |dj|
28
- job.provider_job_id = dj.id
50
+ prepared = Delayed::Backend::JobPreparer.new(JobWrapper.new(job), opts).prepare
51
+ Delayed::Job.new(prepared)
52
+ end
53
+
54
+ def perform_post_enqueue_assignments(active_jobs, delayed_jobs)
55
+ active_jobs.zip(delayed_jobs) do |active_job, delayed_job|
56
+ active_job.successfully_enqueued = true if active_job.respond_to?(:successfully_enqueued=)
57
+ active_job.provider_job_id = delayed_job.id
29
58
  end
30
59
  end
31
60
 
61
+ def coerce_scheduled_at(value)
62
+ value.is_a?(Numeric) ? Time.at(value) : value # rubocop:disable Rails/TimeZone
63
+ end
64
+
32
65
  def enqueue_after_transaction_commit_enabled?(job)
33
66
  job.class.respond_to?(:enqueue_after_transaction_commit) &&
34
67
  [true, :always].include?(job.class.enqueue_after_transaction_commit)
@@ -14,11 +14,37 @@ module Delayed
14
14
 
15
15
  def enqueue_job(options)
16
16
  new(options).tap do |job|
17
- Delayed.lifecycle.run_callbacks(:enqueue, job) do
18
- job.hook(:enqueue)
19
- Delayed::Worker.delay_job?(job) ? job.save : job.invoke_job
17
+ assert_payload_not_active_job!(job.payload_object)
18
+ assert_no_enqueue_hook!(job.payload_object)
19
+ assert_delay_jobs_not_proc!
20
+
21
+ Delayed.lifecycle.run_callbacks(:enqueue, [job]) do
22
+ Delayed::Worker.delay_jobs ? job.save : job.invoke_job
23
+ end
24
+ end
25
+ end
26
+
27
+ # Bulk-enqueue an array of pre-built Delayed::Job instances. Callers are
28
+ # responsible for running JobPreparer and instantiating each job via
29
+ # `Delayed::Job.new(...)` before calling this method.
30
+ def enqueue_all(jobs)
31
+ return 0 if jobs.empty?
32
+
33
+ jobs.each do |job|
34
+ assert_payload_not_active_job!(job.payload_object)
35
+ assert_no_enqueue_hook!(job.payload_object)
36
+ end
37
+ assert_delay_jobs_not_proc!
38
+
39
+ Delayed.lifecycle.run_callbacks(:enqueue, jobs) do
40
+ if Delayed::Worker.delay_jobs
41
+ bulk_insert_all(jobs)
42
+ else
43
+ jobs.each(&:invoke_job)
20
44
  end
21
45
  end
46
+
47
+ jobs.size
22
48
  end
23
49
 
24
50
  def reserve(worker, max_run_time = Worker.max_run_time)
@@ -41,6 +67,38 @@ module Delayed
41
67
  warn '[DEPRECATION] `Delayed::Job.work_off` is deprecated. Use `Delayed::Worker.new.work_off instead.'
42
68
  Delayed::Worker.new.work_off(num)
43
69
  end
70
+
71
+ def name_assignable?
72
+ column_names.include?('name')
73
+ end
74
+
75
+ private
76
+
77
+ def bulk_insert_all(jobs)
78
+ now = db_time_now
79
+ jobs.each { |job| job.created_at = job.updated_at = now }
80
+ rows = jobs.map { |job| job.attributes.compact }
81
+ result = insert_all(rows) # rubocop:disable Rails/SkipsModelValidations
82
+ return unless connection.supports_insert_returning?
83
+
84
+ ids = result.rows.map(&:first)
85
+ jobs.zip(ids) { |job, id| job.id = id }
86
+ end
87
+
88
+ def assert_payload_not_active_job!(payload)
89
+ raise "Delayed::Job enqueue methods do not accept ActiveJobs, use perform_later instead" if payload.is_a?(ActiveJob::Base)
90
+ end
91
+
92
+ def assert_no_enqueue_hook!(payload)
93
+ return if payload.is_a?(Delayed::JobWrapper)
94
+ return unless payload.respond_to?(:enqueue)
95
+
96
+ raise ":enqueue hook on #{payload.class} is no longer supported"
97
+ end
98
+
99
+ def assert_delay_jobs_not_proc!
100
+ raise 'Delayed::Worker.delay_jobs may not be a Proc' if Delayed::Worker.delay_jobs.is_a?(Proc)
101
+ end
44
102
  end
45
103
 
46
104
  attr_reader :error
@@ -58,7 +116,7 @@ module Delayed
58
116
  ParseObjectFromYaml = %r{!ruby/\w+:([^\s]+)} # rubocop:disable Naming/ConstantName
59
117
 
60
118
  def name
61
- if self.class.column_names.include?('name')
119
+ if self.class.name_assignable?
62
120
  super || display_name
63
121
  else
64
122
  display_name # [feat:NameColumn] remove fallback once the "name" column is required.
@@ -105,9 +163,11 @@ module Delayed
105
163
 
106
164
  def hook(name, *args)
107
165
  if payload_object.respond_to?(name)
108
- if payload_object.is_a?(Delayed::JobWrapper)
109
- return if name == :enqueue # this callback is not supported due to method naming conflicts.
166
+ if name == :enqueue
167
+ raise ':enqueue hook is no longer supported'
168
+ end
110
169
 
170
+ if payload_object.is_a?(Delayed::JobWrapper)
111
171
  warn '[DEPRECATION] Job hook methods (`before`, `after`, `success`, etc) are deprecated. Use ActiveJob callbacks instead.'
112
172
  end
113
173
 
@@ -158,17 +218,13 @@ module Delayed
158
218
  @display_name ||= payload_object.display_name if payload_object.respond_to?(:display_name)
159
219
  @display_name ||= payload_object.class.name
160
220
  rescue DeserializationError # [feat:NameColumn] remove this `rescue` once the "name" column is required.
161
- raise if !persisted? && self.class.column_names.include?('name')
221
+ raise if !persisted? && self.class.name_assignable?
162
222
 
163
223
  ParseObjectFromYaml.match(handler)[1]
164
224
  end
165
225
 
166
226
  protected
167
227
 
168
- def set_default_run_at
169
- self.run_at ||= self.class.db_time_now
170
- end
171
-
172
228
  # Call during reload operation to clear out internal state
173
229
  def reset
174
230
  @payload_object = nil
@@ -12,7 +12,10 @@ module Delayed
12
12
  set_payload
13
13
  set_queue_name
14
14
  set_priority
15
+ set_run_at
16
+ set_name
15
17
  handle_dst
18
+ reject_stale_run_at
16
19
  handle_deprecation
17
20
  options
18
21
  end
@@ -33,6 +36,24 @@ module Delayed
33
36
  options[:priority] ||= Delayed::Worker.default_priority
34
37
  end
35
38
 
39
+ def set_run_at
40
+ options[:run_at] ||= Job.db_time_now
41
+ end
42
+
43
+ def set_name
44
+ return if options[:name] || !Job.name_assignable?
45
+
46
+ payload = options[:payload_object]
47
+ options[:name] =
48
+ if payload.respond_to?(:job_data)
49
+ payload.job_data['job_class']
50
+ elsif payload.respond_to?(:display_name)
51
+ payload.display_name
52
+ else
53
+ payload.class.name
54
+ end
55
+ end
56
+
36
57
  def scheduled_into_fall_back_hour?
37
58
  options[:run_at] &&
38
59
  !options[:run_at].in_time_zone.dst? &&
@@ -51,6 +72,18 @@ module Delayed
51
72
  end
52
73
  end
53
74
 
75
+ def reject_stale_run_at
76
+ return unless Delayed::Worker.deny_stale_enqueues
77
+ return unless options[:run_at]
78
+
79
+ threshold = Job.db_time_now - Job.lock_timeout
80
+ return unless options[:run_at] < threshold
81
+
82
+ raise StaleEnqueueError,
83
+ "Cannot enqueue a job in the distant past (run_at: #{options[:run_at].iso8601}," \
84
+ " threshold: #{threshold.iso8601}). This is usually a bug."
85
+ end
86
+
54
87
  def handle_deprecation
55
88
  unless options[:payload_object].respond_to?(:perform)
56
89
  raise ArgumentError,
@@ -14,4 +14,6 @@ module Delayed
14
14
  class FatalBackendError < RuntimeError; end
15
15
 
16
16
  class DeserializationError < StandardError; end
17
+
18
+ class StaleEnqueueError < StandardError; end
17
19
  end
@@ -4,7 +4,7 @@ module Delayed
4
4
  class Lifecycle
5
5
  EVENTS = {
6
6
  execute: [nil],
7
- enqueue: [:job],
7
+ enqueue: [:jobs],
8
8
  perform: %i(worker job),
9
9
  error: %i(worker job),
10
10
  failure: %i(worker job),
@@ -38,7 +38,7 @@ module Delayed
38
38
  def self.sql_now_in_utc
39
39
  case ActiveRecord::Base.connection.adapter_name
40
40
  when 'PostgreSQL'
41
- "TIMEZONE('UTC', NOW())"
41
+ "TIMEZONE('UTC', STATEMENT_TIMESTAMP())"
42
42
  when 'MySQL', 'Mysql2'
43
43
  "UTC_TIMESTAMP()"
44
44
  else
@@ -95,103 +95,129 @@ module Delayed
95
95
  }
96
96
  end
97
97
 
98
- def grouped_count(scope)
99
- Delayed::Job.from(scope.select('priority, queue, COUNT(*) AS count'))
100
- .group(priority_case_statement, :queue).sum(:count)
98
+ # This method generates a query that scans the specified scope, groups by
99
+ # priority and queue, and calculates the specified aggregates. An outer
100
+ # query is executed for priority bucketing and appending db_now_utc (to
101
+ # avoid running these computations for each tuple in the inner query).
102
+ def grouped_query(scope, include_db_time: false, **kwargs)
103
+ inner_selects = kwargs.map { |key, (agg, expr)| as_expression(agg, expr, key) }
104
+ outer_selects = kwargs.map { |key, (agg, _)| as_expression(agg == :count ? :sum : agg, key, key) }
105
+ outer_selects << "#{self.class.sql_now_in_utc} AS db_now_utc" if include_db_time
106
+
107
+ Delayed::Job
108
+ .from(scope.select(:priority, :queue, *inner_selects).group(:priority, :queue))
109
+ .group(priority_case_statement, :queue).select(
110
+ *outer_selects,
111
+ "#{priority_case_statement} AS priority",
112
+ 'queue AS queue',
113
+ ).group_by { |j| [j.priority.to_i, j.queue] }
114
+ .transform_values(&:first)
101
115
  end
102
116
 
103
- def grouped_min(scope, column)
104
- Delayed::Job.from(scope.select("priority, queue, MIN(#{column}) AS #{column}"))
105
- .group(priority_case_statement, :queue)
106
- .select(<<~SQL.squish)
107
- (#{priority_case_statement}) AS priority,
108
- queue,
109
- MIN(#{column}) AS #{column},
110
- #{self.class.sql_now_in_utc} AS db_now_utc
111
- SQL
112
- .group_by { |j| [j.priority.to_i, j.queue] }
113
- .transform_values(&:first)
117
+ def as_expression(aggregate_function, aggregate_expression, column_name)
118
+ "#{aggregate_function.to_s.upcase}(#{aggregate_expression}) AS #{column_name}"
114
119
  end
115
120
 
116
121
  def count_grouped
117
- if Job.connection.supports_partial_index?
118
- failed_count_grouped.merge(live_count_grouped) { |_, l, f| l + f }
119
- else
120
- grouped_count(jobs)
121
- end
122
+ failed_count_grouped.merge(live_count_grouped) { |_, l, f| l + f }
122
123
  end
123
124
 
124
125
  def live_count_grouped
125
- grouped_count(jobs.live)
126
+ live_counts.transform_values(&:count)
126
127
  end
127
128
 
128
129
  def future_count_grouped
129
- grouped_count(jobs.future)
130
+ live_counts.transform_values(&:future_count)
130
131
  end
131
132
 
132
- def locked_count_grouped
133
- @memo[:locked_count_grouped] ||= grouped_count(jobs.claimed)
133
+ def erroring_count_grouped
134
+ live_counts.transform_values(&:erroring_count)
134
135
  end
135
136
 
136
- def erroring_count_grouped
137
- grouped_count(jobs.erroring)
137
+ def locked_count_grouped
138
+ pending_counts.transform_values(&:claimed_count)
138
139
  end
139
140
 
140
141
  def failed_count_grouped
141
- @memo[:failed_count_grouped] ||= grouped_count(jobs.failed)
142
+ failed_counts.transform_values(&:count)
142
143
  end
143
144
 
144
145
  def max_lock_age_grouped
145
- oldest_locked_at_query.transform_values { |j| db_now(j) - j.locked_at }
146
+ pending_counts.transform_values { |j| time_ago(db_now(j), j.locked_at) }
146
147
  end
147
148
 
148
149
  def max_age_grouped
149
- oldest_run_at_query.transform_values { |j| db_now(j) - j.run_at }
150
+ live_counts.transform_values { |j| time_ago(db_now(j), j.run_at) }
150
151
  end
151
152
 
152
153
  def alert_age_percent_grouped
153
- oldest_run_at_query.each_with_object({}) do |((priority, queue), j), metrics|
154
- max_age = db_now(j) - j.run_at
154
+ live_counts.each_with_object({}) do |((priority, queue), j), metrics|
155
+ max_age = time_ago(db_now(j), j.run_at)
155
156
  alert_age = Priority.new(priority).alert_age
156
157
  metrics[[priority, queue]] = [max_age / alert_age * 100, 100].min if alert_age
157
158
  end
158
159
  end
159
160
 
160
161
  def workable_count_grouped
161
- grouped_count(jobs.claimable)
162
+ pending_counts.transform_values(&:claimable_count)
162
163
  end
163
164
 
164
165
  alias working_count_grouped locked_count_grouped
165
166
 
166
167
  def oldest_locked_job_grouped
167
- oldest_locked_at_query.transform_values(&:locked_at)
168
+ pending_counts.transform_values(&:locked_at).compact
168
169
  end
169
170
 
170
171
  def oldest_workable_job_grouped
171
- oldest_run_at_query.transform_values(&:run_at)
172
+ live_counts.transform_values(&:run_at).compact
173
+ end
174
+
175
+ def live_counts
176
+ @memo[:live_counts] ||= grouped_query(
177
+ jobs.live,
178
+ include_db_time: true,
179
+ count: [:count, '*'],
180
+ future_count: [:sum, case_when(Job.future_clause.to_sql)],
181
+ erroring_count: [:sum, case_when(Job.erroring_clause.to_sql)],
182
+ run_at: [:min, case_when(Job.pending_clause.to_sql, 'run_at')],
183
+ )
172
184
  end
173
185
 
174
- def oldest_locked_at_query
175
- @memo[:oldest_locked_at_query] ||= grouped_min(jobs.claimed, :locked_at)
186
+ def pending_counts
187
+ @memo[:pending_counts] ||= grouped_query(
188
+ jobs.pending,
189
+ include_db_time: true,
190
+ claimed_count: [:sum, case_when(Job.claimed_clause.to_sql)],
191
+ claimable_count: [:sum, case_when(Job.claimable_clause.to_sql)],
192
+ locked_at: [:min, case_when(Job.claimed_clause.to_sql, 'locked_at')],
193
+ )
176
194
  end
177
195
 
178
- def oldest_run_at_query
179
- @memo[:oldest_run_at_query] ||= grouped_min(jobs.claimable, :run_at)
196
+ def failed_counts
197
+ @memo[:failed_counts] ||= grouped_query(jobs.failed, count: [:count, '*'])
180
198
  end
181
199
 
182
200
  def db_now(record)
183
201
  self.class.parse_utc_time(record.db_now_utc)
184
202
  end
185
203
 
204
+ def time_ago(now, value)
205
+ [now - (value || now), 0].max
206
+ end
207
+
208
+ def case_when(condition, true_val = 1)
209
+ "CASE WHEN #{condition} THEN #{true_val} ELSE #{true_val == 1 ? 0 : 'NULL'} END"
210
+ end
211
+
186
212
  def priority_case_statement
187
213
  [
188
214
  'CASE',
189
215
  Priority.ranges.values.map do |range|
190
- [
191
- "WHEN priority >= #{range.first.to_i}",
192
- ("AND priority < #{range.last.to_i}" unless range.last.infinite?),
193
- "THEN #{range.first.to_i}",
194
- ].compact
216
+ if range.last.infinite?
217
+ "WHEN priority >= #{range.first.to_i} THEN #{range.first.to_i}"
218
+ else
219
+ "WHEN priority < #{range.last.to_i} THEN #{range.first.to_i}"
220
+ end
195
221
  end,
196
222
  'END',
197
223
  ].flatten.join(' ')
@@ -2,9 +2,9 @@ module Delayed
2
2
  module Plugins
3
3
  class Instrumentation < Plugin
4
4
  callbacks do |lifecycle|
5
- lifecycle.around(:enqueue) do |job, *args, &block|
6
- ActiveSupport::Notifications.instrument('delayed.job.enqueue', active_support_notifications_tags(job)) do
7
- block.call(job, *args)
5
+ lifecycle.around(:enqueue) do |jobs, &block|
6
+ ActiveSupport::Notifications.instrument('delayed.job.enqueue', bulk_enqueue_tags(jobs)) do
7
+ block.call(jobs)
8
8
  end
9
9
  end
10
10
 
@@ -34,6 +34,12 @@ module Delayed
34
34
  job: job,
35
35
  }
36
36
  end
37
+
38
+ def self.bulk_enqueue_tags(jobs)
39
+ {
40
+ jobs: jobs.map { |job| active_support_notifications_tags(job) },
41
+ }
42
+ end
37
43
  end
38
44
  end
39
45
  end
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Delayed
4
- VERSION = '2.1.0'
4
+ VERSION = '3.0.0'
5
5
  end
@@ -22,6 +22,8 @@ module Delayed
22
22
  cattr_accessor :read_ahead, instance_writer: false, default: 5
23
23
  cattr_accessor :destroy_failed_jobs, instance_writer: false, default: false
24
24
 
25
+ cattr_accessor :deny_stale_enqueues, instance_writer: false, default: false
26
+
25
27
  cattr_accessor :min_priority, :max_priority, instance_writer: false
26
28
 
27
29
  # TODO: Remove this and rely on ActiveJob.queue_name when no queue is specified
@@ -35,14 +37,6 @@ module Delayed
35
37
  :default_log_level, :default_log_level=, to: Delayed
36
38
  end
37
39
 
38
- def self.delay_job?(job)
39
- if delay_jobs.is_a?(Proc)
40
- delay_jobs.arity == 1 ? delay_jobs.call(job) : delay_jobs.call
41
- else
42
- delay_jobs
43
- end
44
- end
45
-
46
40
  def initialize
47
41
  @failed_reserve_count = 0
48
42