dalliance 0.6.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.circleci/config.yml +107 -0
- data/Appraisals +8 -4
- data/Gemfile +1 -1
- data/README.rdoc +13 -1
- data/config/locales/en.yml +2 -0
- data/dalliance.gemspec +4 -4
- data/gemfiles/rails_5.0.gemfile +1 -1
- data/gemfiles/rails_5.0.gemfile.lock +107 -80
- data/gemfiles/rails_5.1.gemfile +1 -1
- data/gemfiles/rails_5.1.gemfile.lock +72 -45
- data/gemfiles/rails_5.2.gemfile +1 -1
- data/gemfiles/rails_5.2.gemfile.lock +111 -84
- data/gemfiles/{rails_4.2.gemfile → rails_6.0.gemfile} +2 -2
- data/gemfiles/rails_6.0.gemfile.lock +225 -0
- data/gemfiles/rails_6.1.gemfile +12 -0
- data/gemfiles/rails_6.1.gemfile.lock +233 -0
- data/lib/dalliance.rb +129 -26
- data/lib/dalliance/engine.rb +1 -1
- data/lib/dalliance/progress_meter.rb +5 -5
- data/lib/dalliance/version.rb +1 -1
- data/lib/dalliance/workers/delayed_job.rb +28 -9
- data/lib/dalliance/workers/resque.rb +51 -9
- data/spec/dalliance/asynchronous_delayed_job_spec.rb +42 -0
- data/spec/dalliance/asynchronous_resque_spec.rb +118 -0
- data/spec/dalliance/dalliance_spec.rb +7 -5
- data/spec/dalliance/synchronous_spec.rb +60 -3
- data/spec/spec_helper.rb +2 -0
- data/spec/support/active_record.rb +14 -1
- metadata +57 -12
- data/gemfiles/rails_4.2.gemfile.lock +0 -165
data/lib/dalliance.rb
CHANGED
@@ -91,6 +91,8 @@ module Dalliance
|
|
91
91
|
scope :validation_error, -> { where(:dalliance_status => 'validation_error') }
|
92
92
|
scope :processing_error, -> { where(:dalliance_status => 'processing_error') }
|
93
93
|
scope :completed, -> { where(:dalliance_status => 'completed') }
|
94
|
+
scope :cancel_requested, -> { where(:dalliance_status => 'cancel_requested') }
|
95
|
+
scope :cancelled, -> { where(:dalliance_status => 'cancelled') }
|
94
96
|
|
95
97
|
state_machine :dalliance_status, :initial => :pending do
|
96
98
|
state :pending
|
@@ -98,6 +100,8 @@ module Dalliance
|
|
98
100
|
state :validation_error
|
99
101
|
state :processing_error
|
100
102
|
state :completed
|
103
|
+
state :cancel_requested
|
104
|
+
state :cancelled
|
101
105
|
|
102
106
|
#event :queue_dalliance do
|
103
107
|
# transition :processing_error => :pending
|
@@ -116,7 +120,22 @@ module Dalliance
|
|
116
120
|
end
|
117
121
|
|
118
122
|
event :finish_dalliance do
|
119
|
-
transition :processing => :completed
|
123
|
+
transition [:processing, :cancel_requested] => :completed
|
124
|
+
end
|
125
|
+
|
126
|
+
event :reprocess_dalliance do
|
127
|
+
transition [:validation_error, :processing_error, :completed] => :pending
|
128
|
+
end
|
129
|
+
|
130
|
+
# Requests the record to stop processing. This does NOT cause processing
|
131
|
+
# to stop! Each model is required to handle cancellation on its own by
|
132
|
+
# periodically checking the dalliance status
|
133
|
+
event :request_cancel_dalliance do
|
134
|
+
transition [:pending, :processing] => :cancel_requested
|
135
|
+
end
|
136
|
+
|
137
|
+
event :cancelled_dalliance do
|
138
|
+
transition [:cancel_requested] => :cancelled
|
120
139
|
end
|
121
140
|
end
|
122
141
|
#END state_machine(s)
|
@@ -175,7 +194,9 @@ module Dalliance
|
|
175
194
|
else
|
176
195
|
self.class.where(id: self.id).update_all(dalliance_status: dalliance_status, dalliance_error_hash: dalliance_error_hash.to_yaml )
|
177
196
|
end
|
197
|
+
# rubocop:disable Lint/SuppressedException
|
178
198
|
rescue
|
199
|
+
# rubocop:enable Lint/SuppressedException
|
179
200
|
end
|
180
201
|
end
|
181
202
|
end
|
@@ -185,13 +206,35 @@ module Dalliance
|
|
185
206
|
end
|
186
207
|
|
187
208
|
def error_or_completed?
|
188
|
-
validation_error? || processing_error? || completed?
|
209
|
+
validation_error? || processing_error? || completed? || cancelled?
|
210
|
+
end
|
211
|
+
|
212
|
+
# Cancels the job and removes it from the queue if has not already been taken
|
213
|
+
# by a worker. If the job is processing, it is up to the job implementation
|
214
|
+
# to stop and do any necessary cleanup. If the job does not honor the
|
215
|
+
# cancellation request, it will finish processing as normal and finish with a
|
216
|
+
# dalliance_status of 'completed'.
|
217
|
+
#
|
218
|
+
# Jobs can currently only be removed from Resque queues. DelayedJob jobs will
|
219
|
+
# not be dequeued, but will immediately exit once taken by a worker.
|
220
|
+
def cancel_and_dequeue_dalliance!
|
221
|
+
should_dequeue = pending?
|
222
|
+
|
223
|
+
request_cancel_dalliance!
|
224
|
+
|
225
|
+
if should_dequeue
|
226
|
+
self.dalliance_options[:worker_class].dequeue(self)
|
227
|
+
dalliance_log("[dalliance] #{self.class.name}(#{id}) - #{dalliance_status} - Removed from #{processing_queue} queue")
|
228
|
+
cancelled_dalliance!
|
229
|
+
end
|
230
|
+
|
231
|
+
true
|
189
232
|
end
|
190
233
|
|
191
234
|
def validate_dalliance_status
|
192
235
|
unless error_or_completed?
|
193
|
-
errors.add(:dalliance_status,
|
194
|
-
if defined?(Rails)
|
236
|
+
errors.add(:dalliance_status, "Processing must be finished or cancelled, but status is '#{dalliance_status}'")
|
237
|
+
if defined?(Rails)
|
195
238
|
throw(:abort)
|
196
239
|
else
|
197
240
|
return false
|
@@ -213,17 +256,46 @@ module Dalliance
|
|
213
256
|
end
|
214
257
|
end
|
215
258
|
|
216
|
-
#Force
|
217
|
-
def dalliance_background_process(
|
218
|
-
if
|
219
|
-
self.class.dalliance_options[:worker_class].enqueue(self, processing_queue)
|
259
|
+
#Force background_processing w/ true
|
260
|
+
def dalliance_background_process(background_processing = nil)
|
261
|
+
if background_processing || (background_processing.nil? && self.class.dalliance_options[:background_processing])
|
262
|
+
self.class.dalliance_options[:worker_class].enqueue(self, processing_queue, :dalliance_process)
|
220
263
|
else
|
221
264
|
dalliance_process(false)
|
222
265
|
end
|
223
266
|
end
|
224
267
|
|
225
|
-
|
226
|
-
|
268
|
+
def dalliance_process(background_processing = false)
|
269
|
+
do_dalliance_process(
|
270
|
+
perform_method: self.class.dalliance_options[:dalliance_method],
|
271
|
+
background_processing: background_processing
|
272
|
+
)
|
273
|
+
end
|
274
|
+
|
275
|
+
def dalliance_background_reprocess(background_processing = nil)
|
276
|
+
# Reset state to 'pending' before queueing up
|
277
|
+
# Otherwise the model will stay on completed/processing_error until the job
|
278
|
+
# is taken by a worker, which could be a long time after this method is
|
279
|
+
# called.
|
280
|
+
reprocess_dalliance!
|
281
|
+
if background_processing || (background_processing.nil? && self.class.dalliance_options[:background_processing])
|
282
|
+
self.class.dalliance_options[:worker_class].enqueue(self, processing_queue, :do_dalliance_reprocess)
|
283
|
+
else
|
284
|
+
do_dalliance_reprocess(false)
|
285
|
+
end
|
286
|
+
end
|
287
|
+
|
288
|
+
def dalliance_reprocess(background_processing = false)
|
289
|
+
reprocess_dalliance!
|
290
|
+
do_dalliance_reprocess(background_processing)
|
291
|
+
end
|
292
|
+
|
293
|
+
def do_dalliance_process(perform_method:, background_processing: false)
|
294
|
+
# The job might have been cancelled after it was queued, but before
|
295
|
+
# processing started. Check for that up front before doing any processing.
|
296
|
+
cancelled_dalliance! if cancel_requested?
|
297
|
+
return if cancelled? # method generated from AASM
|
298
|
+
|
227
299
|
start_time = Time.now
|
228
300
|
|
229
301
|
begin
|
@@ -233,9 +305,9 @@ module Dalliance
|
|
233
305
|
build_dalliance_progress_meter(:total_count => calculate_dalliance_progress_meter_total_count).save!
|
234
306
|
end
|
235
307
|
|
236
|
-
self.send(
|
308
|
+
self.send(perform_method)
|
237
309
|
|
238
|
-
finish_dalliance! unless validation_error?
|
310
|
+
finish_dalliance! unless validation_error? || cancelled?
|
239
311
|
rescue StandardError => e
|
240
312
|
#Save the error for future analysis...
|
241
313
|
self.dalliance_error_hash = {:error => e.class.name, :message => e.message, :backtrace => e.backtrace}
|
@@ -255,14 +327,16 @@ module Dalliance
|
|
255
327
|
else
|
256
328
|
self.class.where(id: self.id).update_all(dalliance_status: dalliance_status, dalliance_error_hash: dalliance_error_hash.to_yaml )
|
257
329
|
end
|
330
|
+
# rubocop:disable Lint/SuppressedException
|
258
331
|
rescue
|
332
|
+
# rubocop:enable Lint/SuppressedException
|
259
333
|
end
|
260
334
|
end
|
261
335
|
|
262
336
|
error_notifier.call(e)
|
263
337
|
|
264
|
-
#Don't raise the error if we're
|
265
|
-
raise e unless
|
338
|
+
# Don't raise the error if we're background processing...
|
339
|
+
raise e unless background_processing && self.class.dalliance_options[:worker_class].rescue_error?
|
266
340
|
ensure
|
267
341
|
if self.class.dalliance_options[:dalliance_progress_meter] && dalliance_progress_meter
|
268
342
|
#Works with optimistic locking...
|
@@ -274,8 +348,11 @@ module Dalliance
|
|
274
348
|
|
275
349
|
dalliance_log("[dalliance] #{self.class.name}(#{id}) - #{dalliance_status} #{duration.to_i}")
|
276
350
|
|
277
|
-
|
278
|
-
|
351
|
+
duration_column = self.class.dalliance_options[:duration_column]
|
352
|
+
if duration_column.present?
|
353
|
+
current_duration = self.send(duration_column) || 0
|
354
|
+
self.class.where(id: self.id)
|
355
|
+
.update_all(duration_column => current_duration + duration.to_f)
|
279
356
|
end
|
280
357
|
end
|
281
358
|
end
|
@@ -305,6 +382,19 @@ module Dalliance
|
|
305
382
|
end
|
306
383
|
end
|
307
384
|
|
385
|
+
private
|
386
|
+
|
387
|
+
# Executes the reprocessing method defined in the model's dalliance options.
|
388
|
+
#
|
389
|
+
# @param [Boolean] background_processing
|
390
|
+
# flag if this is called from a background worker. Defaults to false.
|
391
|
+
def do_dalliance_reprocess(background_processing = false)
|
392
|
+
do_dalliance_process(
|
393
|
+
perform_method: self.class.dalliance_options[:reprocess_method],
|
394
|
+
background_processing: background_processing
|
395
|
+
)
|
396
|
+
end
|
397
|
+
|
308
398
|
module Glue
|
309
399
|
extend ActiveSupport::Concern
|
310
400
|
|
@@ -313,15 +403,28 @@ module Dalliance
|
|
313
403
|
end
|
314
404
|
|
315
405
|
module ClassMethods
|
316
|
-
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
406
|
+
# Enables dalliance processing for this class.
|
407
|
+
#
|
408
|
+
# @param [Symbol|String] dalliance_method
|
409
|
+
# the name of the method to call when processing the model in dalliance
|
410
|
+
# @param [Hash] options
|
411
|
+
# an optional hash of options for dalliance processing
|
412
|
+
# @option options [Symbol] :reprocess_method
|
413
|
+
# the name of the method to use to reprocess the model in dalliance
|
414
|
+
# @option options [Boolean] :dalliance_process_meter
|
415
|
+
# whether or not to display a progress meter
|
416
|
+
# @option options [String] :queue
|
417
|
+
# the name of the worker queue to use. Default 'dalliance'
|
418
|
+
# @option options [String] :duration_column
|
419
|
+
# the name of the table column that stores the dalliance processing time. Default 'dalliance_duration'
|
420
|
+
# @option options [Object] :logger
|
421
|
+
# the logger object to use. Can be nil
|
422
|
+
# @option options [Proc] :error_notifier
|
423
|
+
# A proc that accepts an error object. Default is a NOP
|
424
|
+
def dalliance(dalliance_method, options = {})
|
425
|
+
opts = Dalliance.options.merge(options)
|
426
|
+
|
427
|
+
opts[:dalliance_method] = dalliance_method
|
325
428
|
|
326
429
|
if dalliance_options.nil?
|
327
430
|
self.dalliance_options = {}
|
@@ -329,7 +432,7 @@ module Dalliance
|
|
329
432
|
self.dalliance_options = self.dalliance_options.dup
|
330
433
|
end
|
331
434
|
|
332
|
-
self.dalliance_options.merge!(
|
435
|
+
self.dalliance_options.merge!(opts)
|
333
436
|
|
334
437
|
include Dalliance
|
335
438
|
end
|
data/lib/dalliance/engine.rb
CHANGED
@@ -7,7 +7,7 @@ module Dalliance
|
|
7
7
|
ActiveSupport.on_load :active_record do
|
8
8
|
include Dalliance::Glue
|
9
9
|
|
10
|
-
ActiveRecord::ConnectionAdapters::TableDefinition.
|
10
|
+
ActiveRecord::ConnectionAdapters::TableDefinition.include Dalliance::Schema
|
11
11
|
end
|
12
12
|
end
|
13
13
|
end
|
@@ -46,20 +46,20 @@ module Dalliance
|
|
46
46
|
#TODO: This is just a stopgap until I fix increment! to be thread-safe
|
47
47
|
def progress
|
48
48
|
begin
|
49
|
-
|
49
|
+
current_progress = (current_count.to_f / total_count.to_f * 100).to_i
|
50
50
|
|
51
51
|
#Handle an incorrect total_count...
|
52
|
-
|
52
|
+
current_progress = 100 if current_progress > 100
|
53
53
|
rescue
|
54
54
|
#what, are you diving by zero?
|
55
|
-
|
55
|
+
current_progress = 0
|
56
56
|
end
|
57
57
|
|
58
|
-
|
58
|
+
current_progress
|
59
59
|
end
|
60
60
|
|
61
61
|
def increment!
|
62
62
|
Dalliance::ProgressMeter.increment_counter(:current_count, self.id)
|
63
63
|
end
|
64
64
|
end
|
65
|
-
end
|
65
|
+
end
|
data/lib/dalliance/version.rb
CHANGED
@@ -1,15 +1,24 @@
|
|
1
1
|
module Dalliance
|
2
2
|
module Workers
|
3
|
-
if defined?(Rails)
|
3
|
+
if defined?(Rails)
|
4
4
|
class DelayedJob < ::ActiveJob::Base
|
5
5
|
queue_as :dalliance
|
6
6
|
|
7
|
-
def self.enqueue(instance, queue = 'dalliance')
|
8
|
-
Dalliance::Workers::DelayedJob
|
7
|
+
def self.enqueue(instance, queue = 'dalliance', perform_method)
|
8
|
+
Dalliance::Workers::DelayedJob
|
9
|
+
.set(queue: queue)
|
10
|
+
.perform_later(instance.class.name, instance.id, perform_method.to_s)
|
9
11
|
end
|
10
12
|
|
11
|
-
def
|
12
|
-
|
13
|
+
def self.dequeue(_instance)
|
14
|
+
# NOP
|
15
|
+
end
|
16
|
+
|
17
|
+
def perform(instance_klass, instance_id, perform_method)
|
18
|
+
instance_klass
|
19
|
+
.constantize
|
20
|
+
.find(instance_id)
|
21
|
+
.send(perform_method, true)
|
13
22
|
end
|
14
23
|
|
15
24
|
#Delayed job automatically retries, so rescue the error
|
@@ -18,13 +27,23 @@ module Dalliance
|
|
18
27
|
end
|
19
28
|
end
|
20
29
|
else
|
21
|
-
class DelayedJob < Struct.new(:instance_klass, :instance_id)
|
22
|
-
def self.enqueue(instance, queue = 'dalliance')
|
23
|
-
::Delayed::Job.enqueue(
|
30
|
+
class DelayedJob < Struct.new(:instance_klass, :instance_id, :perform_method)
|
31
|
+
def self.enqueue(instance, queue = 'dalliance', perform_method)
|
32
|
+
::Delayed::Job.enqueue(
|
33
|
+
self.new(instance.class.name, instance.id, perform_method),
|
34
|
+
:queue => queue
|
35
|
+
)
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.dequeue(_instance)
|
39
|
+
# NOP
|
24
40
|
end
|
25
41
|
|
26
42
|
def perform
|
27
|
-
instance_klass
|
43
|
+
instance_klass
|
44
|
+
.constantize
|
45
|
+
.find(instance_id)
|
46
|
+
.send(perform_method, true)
|
28
47
|
end
|
29
48
|
|
30
49
|
#Delayed job automatically retries, so rescue the error
|
@@ -1,15 +1,37 @@
|
|
1
1
|
module Dalliance
|
2
2
|
module Workers
|
3
|
-
if defined?(Rails)
|
3
|
+
if defined?(Rails)
|
4
4
|
class Resque < ::ActiveJob::Base
|
5
5
|
queue_as :dalliance
|
6
6
|
|
7
|
-
def self.enqueue(instance, queue = 'dalliance')
|
8
|
-
Dalliance::Workers::Resque
|
7
|
+
def self.enqueue(instance, queue = 'dalliance', perform_method)
|
8
|
+
Dalliance::Workers::Resque
|
9
|
+
.set(queue: queue)
|
10
|
+
.perform_later(instance.class.name, instance.id, perform_method.to_s)
|
9
11
|
end
|
10
12
|
|
11
|
-
def
|
12
|
-
|
13
|
+
def self.dequeue(instance)
|
14
|
+
redis = ::Resque.redis
|
15
|
+
queue = instance.processing_queue
|
16
|
+
|
17
|
+
redis.everything_in_queue(queue).each do |string|
|
18
|
+
# Structure looks like, e.g.
|
19
|
+
# { 'class' => 'ActiveJob::...', 'args' => [{ 'arguments' => ['SomeClass', 123, 'dalliance_process'] }] }
|
20
|
+
data = ::Resque.decode(string)
|
21
|
+
dalliance_args = data['args'][0]['arguments']
|
22
|
+
|
23
|
+
if dalliance_args == [instance.class.name, instance.id, 'dalliance_process'] ||
|
24
|
+
dalliance_args == [instance.class.name, instance.id, 'dalliance_reprocess']
|
25
|
+
redis.remove_from_queue(queue, string)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def perform(instance_klass, instance_id, perform_method)
|
31
|
+
instance_klass
|
32
|
+
.constantize
|
33
|
+
.find(instance_id)
|
34
|
+
.send(perform_method, true)
|
13
35
|
end
|
14
36
|
|
15
37
|
#Resque fails, so don't rescue the error
|
@@ -19,12 +41,32 @@ module Dalliance
|
|
19
41
|
end
|
20
42
|
else
|
21
43
|
class Resque
|
22
|
-
def self.enqueue(instance, queue = 'dalliance')
|
23
|
-
::Resque.enqueue_to(queue, self, instance.class.name, instance.id)
|
44
|
+
def self.enqueue(instance, queue = 'dalliance', perform_method)
|
45
|
+
::Resque.enqueue_to(queue, self, instance.class.name, instance.id, perform_method.to_s)
|
46
|
+
end
|
47
|
+
|
48
|
+
def self.dequeue(instance)
|
49
|
+
redis = ::Resque.redis
|
50
|
+
queue = instance.processing_queue
|
51
|
+
|
52
|
+
redis.everything_in_queue(queue).each do |string|
|
53
|
+
# Structure looks like, e.g.
|
54
|
+
# { 'class' => 'ActiveJob::...', 'args' => [{ 'arguments' => ['SomeClass', 123, 'dalliance_process'] }] }
|
55
|
+
data = ::Resque.decode(string)
|
56
|
+
dalliance_args = data['args'][0]['arguments']
|
57
|
+
|
58
|
+
if dalliance_args == [instance.class.name, instance.id, 'dalliance_process'] ||
|
59
|
+
dalliance_args == [instance.class.name, instance.id, 'dalliance_reprocess']
|
60
|
+
redis.remove_from_queue(queue, string)
|
61
|
+
end
|
62
|
+
end
|
24
63
|
end
|
25
64
|
|
26
|
-
def self.perform(instance_klass, instance_id)
|
27
|
-
instance_klass
|
65
|
+
def self.perform(instance_klass, instance_id, perform_method)
|
66
|
+
instance_klass
|
67
|
+
.constantize
|
68
|
+
.find(instance_id)
|
69
|
+
.send(perform_method, true)
|
28
70
|
end
|
29
71
|
|
30
72
|
#Resque fails, so don't rescue the error
|
@@ -77,6 +77,48 @@ RSpec.describe DallianceModel do
|
|
77
77
|
expect(subject.dalliance_duration).not_to eq(nil)
|
78
78
|
end
|
79
79
|
|
80
|
+
context 'reprocess' do
|
81
|
+
before(:all) do
|
82
|
+
DallianceModel.dalliance_options[:dalliance_method] = :dalliance_success_method
|
83
|
+
DallianceModel.dalliance_options[:worker_class] = Dalliance::Workers::DelayedJob
|
84
|
+
DallianceModel.dalliance_options[:queue] = 'dalliance'
|
85
|
+
end
|
86
|
+
|
87
|
+
before do
|
88
|
+
subject.dalliance_process
|
89
|
+
subject.reload
|
90
|
+
end
|
91
|
+
|
92
|
+
it 'successfully runs the dalliance_reprocess method' do
|
93
|
+
subject.dalliance_background_reprocess
|
94
|
+
Delayed::Worker.new(:queues => [:dalliance]).work_off
|
95
|
+
subject.reload
|
96
|
+
|
97
|
+
expect(subject).to be_successful
|
98
|
+
expect(Delayed::Job.count).to eq(0)
|
99
|
+
expect(subject.reprocessed_count).to eq(1)
|
100
|
+
end
|
101
|
+
|
102
|
+
it 'increases the total processing time counter' do
|
103
|
+
original_duration = subject.dalliance_duration
|
104
|
+
subject.dalliance_background_reprocess
|
105
|
+
Delayed::Worker.new(:queues => [:dalliance]).work_off
|
106
|
+
subject.reload
|
107
|
+
|
108
|
+
expect(subject.dalliance_duration).to be_between(original_duration, Float::INFINITY)
|
109
|
+
end
|
110
|
+
|
111
|
+
it "resets the dalliance_status to 'pending'" do
|
112
|
+
subject.update_column(:dalliance_status, 'processing_error')
|
113
|
+
expect { subject.dalliance_background_reprocess }
|
114
|
+
.to change(subject, :dalliance_status)
|
115
|
+
.to('pending')
|
116
|
+
|
117
|
+
Delayed::Worker.new(:queues => [:dalliance]).work_off
|
118
|
+
expect(subject).to be_successful
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
80
122
|
context "another_queue" do
|
81
123
|
let(:queue) { 'dalliance_2'}
|
82
124
|
|