solid_queue-coalesce 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,114 +1,235 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module ActiveJob
4
+ # Lets many +perform_later+ calls with the same key share one pending job. See #coalesce.
4
5
  module Coalescing
5
6
  extend ActiveSupport::Concern
6
7
 
7
- DEFAULT_COALESCE_LIMIT = 100
8
+ COALESCE_ON_CONFLICT_BEHAVIOUR = %i[ discard collect ]
9
+ COALESCE_ON_FULL_BEHAVIOUR = %i[ wait dispatch ]
10
+ DEFAULT_COALESCE_MAX_CALLS = 100
11
+
12
+ # A job created as a collecting job can't run the way its class says now, or has
13
+ # lost the calls it collected. Nothing has run, and the job fails with its calls.
14
+ class CollectedJobMismatch < StandardError; end
15
+
16
+ # One accepted call to a collecting job, as #perform_calls receives it.
17
+ #
18
+ # +id+ is the row's ID, or +nil+ for +perform_now+ and other adapters. +arguments+
19
+ # is the call's deserialized positional array.
20
+ #
21
+ # +job_data+ is the call's Active Job payload as it was at enqueue, frozen at the
22
+ # top level. A call that never went through the queue passes a Proc, so only
23
+ # reading it needs the arguments to be serializable.
24
+ class Call
25
+ attr_reader :id, :arguments
26
+
27
+ def initialize(id:, arguments:, job_data:)
28
+ @id = id
29
+ @arguments = arguments
30
+ @job_data = job_data
31
+ end
32
+
33
+ def job_data
34
+ @job_data = @job_data.call if @job_data.respond_to?(:call)
35
+ @job_data.freeze
36
+ end
37
+ end
8
38
 
9
39
  # Sits in front of the job's own #perform, whichever class defines it, so
10
- # #arguments can keep holding this call's own arguments.
11
- module CollectedPerform
40
+ # a collecting job runs #perform_calls instead.
41
+ module CollectedPerform # :nodoc:
12
42
  def perform(*)
13
- super(@coalesced_calls)
43
+ perform_calls(coalesced_calls)
14
44
  end
15
45
  end
16
46
 
17
47
  included do
18
48
  class_attribute :coalesce_key, instance_accessor: false
19
- class_attribute :coalesce_collect, default: false, instance_accessor: false
20
- class_attribute :coalesce_limit, instance_accessor: false
21
- class_attribute :coalesce_flush_when_full, instance_accessor: false
22
-
23
- attr_writer :coalesced
49
+ class_attribute :coalesce_on_conflict, instance_accessor: false
50
+ class_attribute :coalesce_max_calls, instance_accessor: false
51
+ class_attribute :coalesce_on_full, instance_accessor: false
52
+ class_attribute :coalesce_debounce, instance_accessor: false
53
+ class_attribute :coalesce_max_wait, instance_accessor: false
24
54
  end
25
55
 
26
- class_methods do
27
- def coalesce(key:, collect: false, limit: nil, flush_when_full: nil)
56
+ # What happened to this call: +:created+, +:discarded+, +:collected+, or +nil+ if
57
+ # this instance has no recorded admission. It isn't serialized, so it's +nil+ on
58
+ # a worker's instance.
59
+ #
60
+ # job = ReindexProductJob.perform_later(product.id)
61
+ # job.coalesce_status # => :discarded
62
+ attr_reader :coalesce_status
63
+
64
+ # The Call objects of this run, oldest first, for perform callbacks to read.
65
+ # It's +nil+ for a job that doesn't collect.
66
+ attr_reader :coalesced_calls
67
+
68
+ private attr_writer :coalesce_status
69
+
70
+ module ClassMethods
71
+ # Shares one pending job between the +perform_later+ calls with the same key. The
72
+ # key is held until that job starts, so a call during a run opens a new job.
73
+ #
74
+ # class ReindexProductJob < ApplicationJob
75
+ # coalesce key: ->(product_id) { product_id }, debounce: 30.seconds, max_wait: 5.minutes
76
+ #
77
+ # def perform(product_id)
78
+ # Product.find(product_id).reindex!
79
+ # end
80
+ # end
81
+ #
82
+ # ==== Options
83
+ #
84
+ # * +:key+ - Required. A String or Symbol literal, or a Proc that receives the job's
85
+ # arguments. It's scoped to the job class, and built like the key of +limits_concurrency+.
86
+ # * +:on_conflict+ - +:discard+, the default, suppresses a call that finds a pending
87
+ # job. +:collect+ stores every call for #perform_calls.
88
+ # * +:max_calls+ - The most calls one run takes with +:collect+. Defaults to 100.
89
+ # * +:on_full+ - With +:collect+, +:wait+, the default, leaves a full job to wait for
90
+ # its time. +:dispatch+ makes it ready as it fills.
91
+ # * +:debounce+ - A duration. Each call pushes a pending job back to that long from now.
92
+ # * +:max_wait+ - A duration greater than +:debounce+ that caps how far a job can move.
93
+ #
94
+ # An invalid option raises +ArgumentError+. The README has recipes for each of these:
95
+ # https://github.com/dtcristo/solid_queue-coalesce#recipes
96
+ def coalesce(key:, on_conflict: :discard, max_calls: nil, on_full: nil, debounce: nil, max_wait: nil)
28
97
  unless (key.is_a?(String) || key.is_a?(Symbol) || key.is_a?(Proc)) && key.present?
29
98
  raise ArgumentError, "`key` must be a non-blank String or Symbol, or a Proc"
30
99
  end
31
100
 
32
- if limit && !collect
33
- raise ArgumentError, "`limit` needs `collect: true`"
101
+ unless on_conflict.in?(COALESCE_ON_CONFLICT_BEHAVIOUR)
102
+ raise ArgumentError, "`on_conflict` must be :discard or :collect"
34
103
  end
35
104
 
36
- if !flush_when_full.nil? && !collect
37
- raise ArgumentError, "`flush_when_full` needs `collect: true`"
38
- end
105
+ if on_conflict == :collect
106
+ max_calls = DEFAULT_COALESCE_MAX_CALLS if max_calls.nil?
39
107
 
40
- limit ||= DEFAULT_COALESCE_LIMIT if collect
108
+ unless max_calls.is_a?(Integer) && max_calls.positive?
109
+ raise ArgumentError, "`max_calls` must be a positive integer"
110
+ end
111
+
112
+ on_full = :wait if on_full.nil?
113
+
114
+ unless on_full.in?(COALESCE_ON_FULL_BEHAVIOUR)
115
+ raise ArgumentError, "`on_full` must be :wait or :dispatch"
116
+ end
117
+ elsif !max_calls.nil?
118
+ raise ArgumentError, "`max_calls` needs `on_conflict: :collect`"
119
+ elsif !on_full.nil?
120
+ raise ArgumentError, "`on_full` needs `on_conflict: :collect`"
121
+ end
41
122
 
42
- if limit && !(limit.is_a?(Integer) && limit.positive?)
43
- raise ArgumentError, "`limit` must be a positive integer"
123
+ unless debounce.nil? || (debounce.is_a?(ActiveSupport::Duration) && debounce.positive?)
124
+ raise ArgumentError, "`debounce` must be a positive duration"
44
125
  end
45
126
 
46
- flush_when_full = true if collect && flush_when_full.nil?
127
+ unless max_wait.nil?
128
+ raise ArgumentError, "`max_wait` needs `debounce`" if debounce.nil?
47
129
 
48
- unless flush_when_full.nil? || flush_when_full == true || flush_when_full == false
49
- raise ArgumentError, "`flush_when_full` must be true or false"
130
+ unless max_wait.is_a?(ActiveSupport::Duration) && max_wait > debounce
131
+ raise ArgumentError, "`max_wait` must be a duration greater than `debounce`"
132
+ end
50
133
  end
51
134
 
52
- ensure_coalesce_compatible_with_concurrency_controls(collect, concurrency_on_conflict)
135
+ ensure_coalesce_compatible_with_concurrency_controls(on_conflict, concurrency_on_conflict)
53
136
 
54
137
  self.coalesce_key = key
55
- self.coalesce_collect = collect
56
- self.coalesce_limit = limit
57
- self.coalesce_flush_when_full = flush_when_full
138
+ self.coalesce_on_conflict = on_conflict
139
+ self.coalesce_max_calls = max_calls
140
+ self.coalesce_on_full = on_full
141
+ self.coalesce_debounce = debounce
142
+ self.coalesce_max_wait = max_wait
58
143
  end
59
144
 
60
- def limits_concurrency(**)
61
- super.tap { ensure_coalesce_compatible_with_concurrency_controls(coalesce_collect, concurrency_on_conflict) }
145
+ def coalesce_collect? # :nodoc:
146
+ coalesce_on_conflict == :collect
147
+ end
148
+
149
+ def limits_concurrency(**) # :nodoc:
150
+ super.tap { ensure_coalesce_compatible_with_concurrency_controls(coalesce_on_conflict, concurrency_on_conflict) }
62
151
  end
63
152
 
64
153
  private
65
154
  # A conflict destroys the job, and with it the calls it has collected
66
- def ensure_coalesce_compatible_with_concurrency_controls(collect, on_conflict)
67
- if collect && on_conflict == :discard
68
- raise ArgumentError, "`coalesce` with `collect: true` can't be combined with `limits_concurrency` with `on_conflict: :discard`"
155
+ def ensure_coalesce_compatible_with_concurrency_controls(coalesce_on_conflict, concurrency_on_conflict)
156
+ if coalesce_on_conflict == :collect && concurrency_on_conflict == :discard
157
+ raise ArgumentError, "`coalesce` with `on_conflict: :collect` can't be combined with `limits_concurrency` with `on_conflict: :discard`"
69
158
  end
70
159
  end
71
160
  end
72
161
 
73
- # Runs before the arguments are deserialized. Raising here reaches the rescue
74
- # in ClaimedExecution#execute, which turns it into a failed execution.
75
- def perform_now
162
+ # Runs before the arguments are deserialized, and outside the rescue_from of
163
+ # super, so retry_on can't swallow what's raised here. It reaches the rescue in
164
+ # ClaimedExecution#execute, which turns it into a failed execution.
165
+ def perform_now # :nodoc:
76
166
  start_coalesced_run
77
167
  super
78
168
  end
79
169
 
80
- # Scoped to the job class name, and built like concurrency_key
81
- def coalesce_key
170
+ # How the job was created is stored with the job, so it survives retries and
171
+ # outlives the class's declaration. Only the job's row starts with this, when it
172
+ # gets its key, so an instance only has it if it was read from one.
173
+ def serialize # :nodoc:
174
+ @coalesce_on_conflict ? super.merge("coalesce_on_conflict" => @coalesce_on_conflict) : super
175
+ end
176
+
177
+ def deserialize(job_data) # :nodoc:
178
+ super
179
+ @coalesce_on_conflict = job_data["coalesce_on_conflict"]
180
+ end
181
+
182
+ # Scoped to the job class name, and built like concurrency_key. In a batch it's
183
+ # scoped to the batch too, so each coalesced job belongs to one batch, and a call
184
+ # that joins it can't leave a batch waiting on another batch's job.
185
+ def coalesce_key # :nodoc:
82
186
  if self.class.coalesce_key
83
187
  param = compute_concurrency_parameter(self.class.coalesce_key)
84
188
 
85
189
  case param
86
190
  when ActiveRecord::Base
87
- [ self.class.name, param.class.name, param.id ]
191
+ [ coalesce_batch_scope, self.class.name, param.class.name, param.id ]
88
192
  else
89
- [ self.class.name, param ]
193
+ [ coalesce_batch_scope, self.class.name, param ]
90
194
  end.compact.join("/")
91
195
  end
92
196
  end
93
197
 
94
- # Whether this call joined a job that was already open
95
- def coalesced?
96
- !!@coalesced
198
+ # Runs a collecting job with its Call objects, oldest first, instead of +perform+.
199
+ # Define it on a job with <tt>on_conflict: :collect</tt>. A retry runs all of the
200
+ # calls again, so it has to be idempotent.
201
+ #
202
+ # def perform_calls(calls)
203
+ # SearchIndex.bulk_index(Product.where(id: calls.map { |call| call.arguments.last }))
204
+ # end
205
+ def perform_calls(_calls)
206
+ raise NotImplementedError, "#{self.class.name} uses `coalesce` with `on_conflict: :collect`, so it needs a `perform_calls(calls)` method"
97
207
  end
98
208
 
99
209
  # A retry is the same work continuing, so it can't join or suppress anything
100
- def coalesced_enqueue?
210
+ def coalesced_enqueue? # :nodoc:
101
211
  self.class.coalesce_key.present? && executions.to_i.zero?
102
212
  end
103
213
 
214
+ # A call's own time if it has one, and otherwise now plus the window. A longer
215
+ # wait is capped without raising, because a scheduling preference shouldn't fail
216
+ # the caller's request.
217
+ def coalesced_scheduled_at(scheduled_at) # :nodoc:
218
+ scheduled_at ||= self.class.coalesce_debounce&.from_now || Time.current
219
+
220
+ self.class.coalesce_max_wait ? [ scheduled_at, self.class.coalesce_max_wait.from_now ].min : scheduled_at
221
+ end
222
+
104
223
  # The stored key, in case the arguments it was computed from are gone
105
- def concurrency_key
224
+ def concurrency_key # :nodoc:
106
225
  @coalesced_arguments_gone ? @coalesced_run["concurrency_key"] : super
107
226
  end
108
227
 
109
228
  # A retry that doesn't get enqueued has to fail the run: finishing would
110
229
  # destroy the job and the calls it owns, with no job left to process them.
111
- def retry_job(options = {})
230
+ def retry_job(options = {}) # :nodoc:
231
+ ensure_retry_can_take_calls
232
+
112
233
  super.tap do |enqueued|
113
234
  if @coalesced_run && !enqueued
114
235
  raise enqueue_error || SolidQueue::Job::EnqueueError.new("Couldn't enqueue the retry of #{self.class.name} (#{job_id})")
@@ -117,18 +238,50 @@ module ActiveJob
117
238
  end
118
239
 
119
240
  private
241
+ def coalesce_batch_scope
242
+ [ SolidQueue::Batch.name, batch_id ] if batch_id
243
+ end
244
+
120
245
  # Only the run of a job a worker has claimed. perform_now on an instance that
121
- # went through perform_later also has a provider_job_id.
246
+ # went through perform_later also has a provider_job_id. A collecting job always
247
+ # looks for its run, because the marker outlives the class's key and adapter.
122
248
  def start_coalesced_run
123
- if self.class.coalesce_key && provider_job_id && self.class.queue_adapter_name == "solid_queue"
249
+ ensure_still_collecting if created_collecting?
250
+
251
+ if created_collecting? || (self.class.coalesce_key && provider_job_id && self.class.queue_adapter_name == "solid_queue")
124
252
  @coalesced_run = SolidQueue::Job.joins(:claimed_execution).find_by(id: provider_job_id)&.start_coalesced_run
253
+ ensure_calls_present if created_collecting?
254
+ end
255
+ end
256
+
257
+ def created_collecting?
258
+ @coalesce_on_conflict == "collect"
259
+ end
260
+
261
+ # Only Solid Queue moves the calls to the retry. On another adapter the retry
262
+ # would succeed, and they'd stay behind on a job that goes on to finish.
263
+ def ensure_retry_can_take_calls
264
+ if worker_coalesced_rows.any? && self.class.queue_adapter_name != "solid_queue"
265
+ raise CollectedJobMismatch, "#{self.class.name} (#{job_id}) owns collected calls, which can't move to a retry on the `#{self.class.queue_adapter_name}` adapter. Its calls are kept: restore the Solid Queue adapter and retry this failed job, or discard the job to delete them."
266
+ end
267
+ end
268
+
269
+ def ensure_calls_present
270
+ if worker_coalesced_rows.empty?
271
+ raise CollectedJobMismatch, "#{self.class.name} (#{job_id}) was enqueued with `coalesce on_conflict: :collect`, but it has no calls to run: either something other than Solid Queue deleted them, or this worker no longer has the job's claim. Its own arguments are only a copy of its first call, so it won't run them. If its calls are gone, discard this failed job."
272
+ end
273
+ end
274
+
275
+ def ensure_still_collecting
276
+ unless self.class.coalesce_collect?
277
+ raise CollectedJobMismatch, "#{self.class.name} (#{job_id}) was enqueued with `coalesce on_conflict: :collect`, but its class no longer has that. Its calls are kept: restore the declaration and retry this failed job, or discard the job to delete them."
125
278
  end
126
279
  end
127
280
 
128
281
  def _perform_job
129
- return super unless self.class.coalesce_collect
282
+ return super unless self.class.coalesce_collect?
130
283
 
131
- @coalesced_calls = coalesced_calls
284
+ @coalesced_calls = load_coalesced_calls
132
285
 
133
286
  if @coalesced_calls.any?
134
287
  extend CollectedPerform
@@ -151,37 +304,43 @@ module ActiveJob
151
304
  @coalesced_run&.fetch("calls") || []
152
305
  end
153
306
 
154
- def coalesced_calls
307
+ def load_coalesced_calls
155
308
  if worker_coalesced_rows.any?
156
309
  calls_from_coalesced_rows
157
310
  else
158
- [ coalesced_call_from(arguments) ]
311
+ [ Call.new(id: nil, arguments: arguments, job_data: -> { serialize }) ]
159
312
  end
160
313
  end
161
314
 
162
- # Anything but a deleted record raises on the way, before any row has been
163
- # dropped, and fails the run with its rows. When every record is gone the rows
164
- # stay until the job is destroyed: if this run crashes before finishing, the
165
- # re-run needs them to tell it has nothing to do, because a job without rows
166
- # runs its own arguments.
315
+ # Calls whose records are gone are skipped for this attempt. Every row stays
316
+ # until its job is destroyed, and the next attempt deserializes them again.
317
+ # Any other error fails the run before skipped calls have been reported.
167
318
  def calls_from_coalesced_rows
168
- loaded, missing = worker_coalesced_rows.map { |id, serialized| [ id, load_coalesced_row(serialized) ] }.partition(&:last)
169
- drop_coalesced_rows(missing, delete: loaded.any?)
319
+ missing = {}
320
+
321
+ calls = worker_coalesced_rows.filter_map do |id, job_data|
322
+ Call.new(id: id, arguments: deserialize_call_arguments(job_data), job_data: job_data)
323
+ rescue ActiveJob::DeserializationError => error
324
+ raise unless record_not_found?(error)
325
+
326
+ missing[id] = error
327
+ nil
328
+ end
170
329
 
171
- loaded.map { |_, call_arguments| coalesced_call_from(call_arguments) }
330
+ instrument_skipped_calls(missing)
331
+
332
+ calls
172
333
  end
173
334
 
174
- def load_coalesced_row(serialized)
175
- ActiveJob::Arguments.deserialize(serialized)
176
- rescue ActiveJob::DeserializationError => error
177
- raise unless record_not_found?(error)
335
+ # On an instance of its own, as in a normal run, so the job's overrides apply.
336
+ # That instance has no rows, so its deserialize_arguments_if_needed lets errors out.
337
+ def deserialize_call_arguments(job_data)
338
+ self.class.deserialize(job_data.dup).tap { |call| call.send(:deserialize_arguments_if_needed) }.arguments
178
339
  end
179
340
 
180
- def drop_coalesced_rows(rows, delete:)
181
- rows.each do |id, _|
182
- SolidQueue.instrument(:drop_coalesced_argument, argument_id: id, job_id: provider_job_id, active_job_id: job_id) do
183
- SolidQueue::CoalescedArgument.where(id: id).delete_all if delete
184
- end
341
+ def instrument_skipped_calls(rows)
342
+ rows.each do |id, error|
343
+ SolidQueue.instrument(:drop_coalesced_call, call_id: id, job_id: provider_job_id, active_job_id: job_id, class_name: self.class.name, exception_class: error.class.name)
185
344
  end
186
345
  end
187
346
 
@@ -190,9 +349,5 @@ module ActiveJob
190
349
  error.cause.is_a?(ActiveRecord::RecordNotFound) ||
191
350
  (defined?(ActiveJob::DeserializationError::RecordNotFound) && error.is_a?(ActiveJob::DeserializationError::RecordNotFound))
192
351
  end
193
-
194
- def coalesced_call_from(call_arguments)
195
- call_arguments.size == 1 ? call_arguments.first : call_arguments
196
- end
197
352
  end
198
353
  end
@@ -1,10 +1,9 @@
1
1
  class CreateSolidQueueCoalesceTables < ActiveRecord::Migration[7.1]
2
2
  def change
3
3
  create_table :solid_queue_coalesce_keys do |t|
4
- # MySQL's default collations treat keys that differ by case or accent as the
5
- # same key, which would drop a call that should open its own job
6
- t.string :key, null: false, collation: ("utf8mb4_bin" if connection.adapter_name.match?(/mysql|trilogy/i))
4
+ t.string :key, null: false
7
5
  t.bigint :job_id
6
+ t.integer :calls_count, null: false, default: 0
8
7
  t.datetime :created_at, null: false
9
8
 
10
9
  t.index :key, unique: true
@@ -12,16 +11,12 @@ class CreateSolidQueueCoalesceTables < ActiveRecord::Migration[7.1]
12
11
  t.foreign_key :solid_queue_jobs, column: :job_id, on_delete: :cascade
13
12
  end
14
13
 
15
- create_table :solid_queue_coalesced_arguments do |t|
16
- t.bigint :coalesce_key_id
17
- t.bigint :job_id
14
+ create_table :solid_queue_coalesced_calls do |t|
15
+ t.bigint :job_id, null: false
18
16
  t.text :arguments, null: false
19
- t.datetime :scheduled_at
20
17
  t.datetime :created_at, null: false
21
18
 
22
- t.index :coalesce_key_id
23
19
  t.index :job_id
24
- t.foreign_key :solid_queue_coalesce_keys, column: :coalesce_key_id, on_delete: :cascade
25
20
  t.foreign_key :solid_queue_jobs, column: :job_id, on_delete: :cascade
26
21
  end
27
22
  end
@@ -2,7 +2,7 @@
2
2
 
3
3
  module SolidQueue
4
4
  module Coalesce
5
- class Engine < ::Rails::Engine
5
+ class Engine < ::Rails::Engine # :nodoc:
6
6
  initializer "solid_queue.coalesce.active_job", after: "solid_queue.active_job.extensions" do
7
7
  ActiveSupport.on_load :active_job do
8
8
  include ActiveJob::Coalescing
@@ -4,7 +4,7 @@
4
4
  # details of the version pinned in the gemspec.
5
5
  module SolidQueue
6
6
  module Coalesce
7
- module Patches
7
+ module Patches # :nodoc: all
8
8
  # Prepended rather than added to the adapter, because RecurringTask calls
9
9
  # Job.enqueue directly.
10
10
  module Enqueuing
@@ -12,19 +12,26 @@ module SolidQueue
12
12
  coalesced_jobs, other_jobs = active_jobs.partition(&:coalesced_enqueue?)
13
13
  return super if coalesced_jobs.empty?
14
14
 
15
- coalesced_jobs.each { |active_job| active_job.scheduled_at ||= Time.current }
16
- ensure_coalescible(coalesced_jobs)
15
+ # Bulk enqueues bypass ActiveJob#enqueue, and the key depends on the batch
16
+ coalesced_jobs.each do |active_job|
17
+ active_job.scheduled_at = active_job.coalesced_scheduled_at(active_job.scheduled_at)
18
+ active_job.batch_id = Batch.current_batch_id || active_job.batch_id
19
+ end
20
+ ensure_coalescible
21
+
22
+ prepared_calls = prepare_all_coalesced(coalesced_jobs)
17
23
 
18
24
  super(other_jobs) if other_jobs.any?
19
- enqueue_all_coalesced(coalesced_jobs)
25
+ enqueue_all_coalesced(prepared_calls)
20
26
 
21
27
  active_jobs.count(&:successfully_enqueued?)
22
28
  end
23
29
 
24
- def enqueue(active_job, scheduled_at: Time.current)
30
+ # Only the adapter's enqueue_at passes a time, so a plain call passes none
31
+ def enqueue(active_job, **options)
25
32
  if active_job.coalesced_enqueue?
26
- active_job.scheduled_at = scheduled_at
27
- ensure_coalescible([ active_job ])
33
+ active_job.scheduled_at = active_job.coalesced_scheduled_at(options[:scheduled_at])
34
+ ensure_coalescible
28
35
  enqueue_coalesced(active_job.coalesce_key, [ active_job ])
29
36
  elsif active_job.class.coalesce_key
30
37
  continuing_coalesced_run(active_job) { super }
@@ -36,6 +43,8 @@ module SolidQueue
36
43
 
37
44
  module Failing
38
45
  def failed_with(exception)
46
+ return super unless may_hold_coalesce_key?
47
+
39
48
  FailedExecution.transaction(requires_new: true) do
40
49
  release_coalesce_key
41
50
  super
@@ -43,19 +52,46 @@ module SolidQueue
43
52
  end
44
53
  end
45
54
 
46
- # Maintenance#start runs its steps from an inline block, so this follows
47
- # the only concurrency step it can reach.
48
- module Maintaining
55
+ # Before super, so the key is locked before the job, as in admission. A key
56
+ # always means a live job: one left on a job that finished without starting,
57
+ # because its class stopped coalescing, would accept calls that never run. A
58
+ # job that isn't preserved gets destroyed, and the destroy callback releases it.
59
+ module Finishing
60
+ def finished!
61
+ release_coalesce_key if SolidQueue.preserve_finished_jobs?
62
+ super
63
+ end
64
+ end
65
+
66
+ # A class can start discarding conflicts after it stopped collecting, and a
67
+ # conflict destroys the job. A collecting job blocks instead, whatever its class
68
+ # says now, so it reaches a worker and fails with its calls.
69
+ module ConcurrencyConflicts
49
70
  private
50
- def expire_semaphores
51
- super
52
- release_stranded_coalesce_keys
71
+ def handle_concurrency_conflict
72
+ arguments["coalesce_on_conflict"] == "collect" ? block : super
53
73
  end
74
+ end
54
75
 
55
- def release_stranded_coalesce_keys
56
- wrap_in_app_executor do
57
- CoalesceKey.release_stranded(batch_size) if CoalesceKey.migrated?
58
- end
76
+ # Prepended to each execution class that can be discarded, rather than to their
77
+ # parent, because ReadyExecution signals semaphores in its own discard_jobs.
78
+ # Admission locks the key, then the job and its semaphore, so this has to lock
79
+ # the key before either. Left to the cascade, they'd deadlock. On MySQL they
80
+ # still can, because the locks on the executions also cover the gaps where
81
+ # admission inserts its own, so these get retried like admission.
82
+ module BulkDiscarding
83
+ def discard_all_in_batches(...)
84
+ Job.retrying_deadlocks { super }
85
+ end
86
+
87
+ def discard_all_from_jobs(...)
88
+ Job.retrying_deadlocks { super }
89
+ end
90
+
91
+ private
92
+ def discard_jobs(job_ids)
93
+ CoalesceKey.release_all(job_ids) if CoalesceKey.migrated?
94
+ super
59
95
  end
60
96
  end
61
97
 
@@ -64,14 +100,23 @@ module SolidQueue
64
100
 
65
101
  included do
66
102
  validate :ensure_job_class_not_collecting
103
+ validate :ensure_job_class_not_debounced
67
104
  end
68
105
 
69
106
  private
70
107
  # A tick that joins an open job would record that job again, and the unique
71
108
  # index on recurring_executions.job_id would roll back the call it collected.
72
109
  def ensure_job_class_not_collecting
73
- if class_name.present? && job_class&.try(:coalesce_collect)
74
- errors.add :class_name, :collecting, message: "can't be a job that uses `coalesce` with `collect: true`"
110
+ if class_name.present? && job_class&.try(:coalesce_collect?)
111
+ errors.add :class_name, :collecting, message: "can't be a job that uses `coalesce` with `on_conflict: :collect`"
112
+ end
113
+ end
114
+
115
+ # A tick passes no time, so the window applies, and a tick that joins pushes
116
+ # the job back. With an interval shorter than the window it would never run.
117
+ def ensure_job_class_not_debounced
118
+ if class_name.present? && job_class&.try(:coalesce_debounce)
119
+ errors.add :class_name, :debounced, message: "can't be a job that uses `coalesce` with `debounce`"
75
120
  end
76
121
  end
77
122
  end
@@ -80,19 +125,24 @@ module SolidQueue
80
125
  SolidQueue::Job.include SolidQueue::Job::Coalescible
81
126
  SolidQueue::Job.singleton_class.prepend Enqueuing
82
127
  SolidQueue::Job.prepend Failing
128
+ SolidQueue::Job.prepend Finishing
129
+ SolidQueue::Job.prepend ConcurrencyConflicts
130
+ [ ReadyExecution, ScheduledExecution, BlockedExecution, FailedExecution ].each { |execution| execution.singleton_class.prepend BulkDiscarding }
83
131
  SolidQueue::RecurringTask.include RecurringValidation
84
- SolidQueue::Dispatcher::Maintenance.prepend Maintaining
85
132
  end
86
133
  end
87
134
  end
88
135
  end
89
136
 
90
137
  class SolidQueue::LogSubscriber
91
- def coalesce(event)
92
- debug formatted_event(event, action: "Coalesce call", **event.payload.slice(:key, :job_id, :active_job_id), outcome: event.payload[:action])
138
+ # This one fires for every call, so above debug it shouldn't build a line for each
139
+ subscribe_log_level :coalesce, :debug
140
+
141
+ def coalesce(event) # :nodoc:
142
+ debug formatted_event(event, action: "Coalesce call", **event.payload.slice(:job_id, :active_job_id, :class_name), outcome: event.payload[:status])
93
143
  end
94
144
 
95
- def drop_coalesced_argument(event)
96
- warn formatted_event(event, action: "Drop coalesced call for a deleted record", **event.payload.slice(:argument_id, :job_id, :active_job_id))
145
+ def drop_coalesced_call(event) # :nodoc:
146
+ warn formatted_event(event, action: "Skip coalesced call for a deleted record", **event.payload.slice(:call_id, :job_id, :active_job_id, :class_name, :exception_class))
97
147
  end
98
148
  end
@@ -1,5 +1,5 @@
1
1
  module SolidQueue
2
2
  module Coalesce
3
- VERSION = "0.1.0"
3
+ VERSION = "0.2.0"
4
4
  end
5
5
  end