acidic_job 1.0.0.pre11 → 1.0.0.pre14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.tool-versions +1 -0
- data/Gemfile.lock +1 -1
- data/README.md +13 -6
- data/lib/acidic_job/awaiting.rb +22 -37
- data/lib/acidic_job/extensions/action_mailer.rb +2 -7
- data/lib/acidic_job/extensions/active_job.rb +1 -7
- data/lib/acidic_job/extensions/noticed.rb +4 -9
- data/lib/acidic_job/extensions/sidekiq.rb +3 -8
- data/lib/acidic_job/idempotency_key.rb +17 -9
- data/lib/acidic_job/perform_wrapper.rb +8 -35
- data/lib/acidic_job/staging.rb +1 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +16 -5
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fcc6621d513d4284bb09bdb4da7433b81b2c240ce9873dbf1348569e76942960
|
4
|
+
data.tar.gz: 411b5819c7faab9f8bcb19f39bd4dc5c8a03d42f28239f5f19489b2f7d01eb68
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 644bc4268dfa5c66804e169314f67452b4046bdbbd4b013589b6463e0c79363407d8fbdecfc2db3197cccdb75c558059492b2a9f22a0eca0198f7033836abd05
|
7
|
+
data.tar.gz: 2807478181d478a7ee10e082e7f76f42409e3c587e436712396371d92d8b53481b0606a19bbcb108fa2cb95c4529f4b9ca86b146cf1aec64c53954c57eda5971
|
data/.tool-versions
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby 3.0.2
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# AcidicJob
|
2
2
|
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/acidic_job.svg)](https://badge.fury.io/rb/acidic_job)
|
4
|
+
![main workflow](https://github.com/fractaledmind/acidic_job/actions/workflows/main.yml/badge.svg)
|
5
|
+
|
3
6
|
### Idempotent operations for Rails apps (for ActiveJob or Sidekiq)
|
4
7
|
|
5
8
|
At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`, or we Rubyists can use the tried and true power of pure `Sidekiq`. With either we can easily trigger from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed seamlessly.
|
@@ -193,16 +196,20 @@ class ExampleJob < ActiveJob::Base
|
|
193
196
|
end
|
194
197
|
```
|
195
198
|
|
196
|
-
These options cover the two common situations, but sometimes our systems need finer-grained control. For example, our job might take some record as the job argument, but we need to use a combination of the record identifier and record status as the foundation for the idempotency key. In these cases
|
197
|
-
|
198
|
-
When you call any `deliver_acidicly` or `perform_acidicly` method you can pass an optional `unique_by` argument which will be used to generate the idempotency key:
|
199
|
+
These options cover the two common situations, but sometimes our systems need finer-grained control. For example, our job might take some record as the job argument, but we need to use a combination of the record identifier and record status as the foundation for the idempotency key. In these cases you can pass a `Proc` to an `acidic_by` class method:
|
199
200
|
|
200
201
|
```ruby
|
201
|
-
ExampleJob
|
202
|
-
|
202
|
+
class ExampleJob < ActiveJob::Base
|
203
|
+
include AcidicJob
|
204
|
+
acidic_by ->(record:) { [record.id, record.status] }
|
205
|
+
|
206
|
+
def perform(record:)
|
207
|
+
# the idempotency key will be based on whatever the values of `record.id` and `record.status` are
|
208
|
+
end
|
209
|
+
end
|
203
210
|
```
|
204
211
|
|
205
|
-
|
212
|
+
> **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
|
206
213
|
|
207
214
|
|
208
215
|
### Sidekiq Callbacks
|
data/lib/acidic_job/awaiting.rb
CHANGED
@@ -6,47 +6,32 @@ module AcidicJob
|
|
6
6
|
module Awaiting
|
7
7
|
extend ActiveSupport::Concern
|
8
8
|
|
9
|
-
class_methods do
|
10
|
-
# TODO: Allow the `perform` method to be used to kick off Sidekiq Batch powered workflows
|
11
|
-
def initiate(*args)
|
12
|
-
raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
|
13
|
-
|
14
|
-
top_level_workflow = Sidekiq::Batch.new
|
15
|
-
top_level_workflow.on(:success, self, *args)
|
16
|
-
top_level_workflow.jobs do
|
17
|
-
perform_async
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
9
|
def enqueue_step_parallel_jobs(jobs, run, step_result)
|
23
10
|
# `batch` is available from Sidekiq::Pro
|
24
11
|
raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
|
25
12
|
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
raise TooManyParametersForParallelJob
|
49
|
-
end
|
13
|
+
step_batch = Sidekiq::Batch.new
|
14
|
+
# step_batch.description = "AcidicJob::Workflow Step: #{step}"
|
15
|
+
step_batch.on(
|
16
|
+
:success,
|
17
|
+
"#{self.class.name}#step_done",
|
18
|
+
# NOTE: options are marshalled through JSON so use only basic types.
|
19
|
+
{ "run_id" => run.id,
|
20
|
+
"step_result_yaml" => step_result.to_yaml.strip }
|
21
|
+
)
|
22
|
+
# NOTE: The jobs method is atomic.
|
23
|
+
# All jobs created in the block are actually pushed atomically at the end of the block.
|
24
|
+
# If an error is raised, none of the jobs will go to Redis.
|
25
|
+
step_batch.jobs do
|
26
|
+
jobs.each do |worker_name|
|
27
|
+
# TODO: handle Symbols as well
|
28
|
+
worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
|
29
|
+
if worker.instance_method(:perform).arity.zero?
|
30
|
+
worker.perform_async
|
31
|
+
elsif worker.instance_method(:perform).arity == 1
|
32
|
+
worker.perform_async(run.id)
|
33
|
+
else
|
34
|
+
raise TooManyParametersForParallelJob
|
50
35
|
end
|
51
36
|
end
|
52
37
|
end
|
@@ -7,19 +7,14 @@ module AcidicJob
|
|
7
7
|
module ActionMailer
|
8
8
|
extend ActiveSupport::Concern
|
9
9
|
|
10
|
-
def deliver_acidicly(_options = {}
|
10
|
+
def deliver_acidicly(_options = {})
|
11
11
|
job_class = ::ActionMailer::MailDeliveryJob
|
12
12
|
|
13
13
|
job_args = [@mailer_class.name, @action.to_s, "deliver_now", @params, *@args]
|
14
14
|
# for Sidekiq, this depends on the Sidekiq::Serialization extension
|
15
15
|
serialized_job = job_class.new(job_args).serialize
|
16
16
|
acidic_identifier = job_class.respond_to?(:acidic_identifier) ? job_class.acidic_identifier : :job_id
|
17
|
-
|
18
|
-
key = if unique_by
|
19
|
-
IdempotencyKey.generate(unique_by: unique_by, job_class: job_class.name)
|
20
|
-
else
|
21
|
-
IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
22
|
-
end
|
17
|
+
key = IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
23
18
|
|
24
19
|
AcidicJob::Run.create!(
|
25
20
|
staged: true,
|
@@ -24,13 +24,7 @@ module AcidicJob
|
|
24
24
|
raise UnsupportedExtension unless defined?(::ActiveJob) && self < ::ActiveJob::Base
|
25
25
|
|
26
26
|
serialized_job = serialize_with_arguments(*args, **kwargs)
|
27
|
-
|
28
|
-
key = if kwargs.key?(:unique_by) || kwargs.key?("unique_by")
|
29
|
-
unique_by = [kwargs[:unique_by], kwargs["unique_by"]].compact.first
|
30
|
-
IdempotencyKey.generate(unique_by: unique_by, job_class: name)
|
31
|
-
else
|
32
|
-
IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
33
|
-
end
|
27
|
+
key = IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
34
28
|
|
35
29
|
AcidicJob::Run.create!(
|
36
30
|
staged: true,
|
@@ -6,12 +6,12 @@ module AcidicJob
|
|
6
6
|
extend ActiveSupport::Concern
|
7
7
|
|
8
8
|
class_methods do
|
9
|
-
def deliver_acidicly(recipients
|
10
|
-
new.deliver_acidicly(recipients
|
9
|
+
def deliver_acidicly(recipients)
|
10
|
+
new.deliver_acidicly(recipients)
|
11
11
|
end
|
12
12
|
end
|
13
13
|
|
14
|
-
def deliver_acidicly(recipients
|
14
|
+
def deliver_acidicly(recipients)
|
15
15
|
# THIS IS A HACK THAT COPIES AND PASTES KEY PARTS OF THE `Noticed::Base` CODE
|
16
16
|
# IN ORDER TO ALLOW US TO TRANSACTIONALLY DELIVER NOTIFICATIONS
|
17
17
|
# THIS IS THUS LIABLE TO BREAK WHENEVER THAT GEM IS UPDATED
|
@@ -37,12 +37,7 @@ module AcidicJob
|
|
37
37
|
}
|
38
38
|
serialized_job = job_class.send(:job_or_instantiate, args).serialize
|
39
39
|
acidic_identifier = job_class.respond_to?(:acidic_identifier) ? job_class.acidic_identifier : :job_id
|
40
|
-
|
41
|
-
key = if unique_by
|
42
|
-
IdempotencyKey.generate(unique_by: unique_by, job_class: job_class.name)
|
43
|
-
else
|
44
|
-
IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
45
|
-
end
|
40
|
+
key = IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
46
41
|
|
47
42
|
AcidicJob::Run.create!(
|
48
43
|
staged: true,
|
@@ -34,9 +34,10 @@ module AcidicJob
|
|
34
34
|
end
|
35
35
|
end
|
36
36
|
|
37
|
-
def serialize_job(args
|
37
|
+
def serialize_job(*args, **kwargs)
|
38
38
|
# `@args` is only set via `deserialize`; it is not a standard Sidekiq thing
|
39
39
|
arguments = args || @args
|
40
|
+
arguments += [kwargs] unless kwargs.empty?
|
40
41
|
normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(arguments))
|
41
42
|
item = { "class" => self.class, "args" => normalized_args, "jid" => jid }
|
42
43
|
sidekiq_options = sidekiq_options_hash || {}
|
@@ -58,13 +59,7 @@ module AcidicJob
|
|
58
59
|
class_methods do
|
59
60
|
def perform_acidicly(*args, **kwargs)
|
60
61
|
serialized_job = serialize_with_arguments(*args, **kwargs)
|
61
|
-
|
62
|
-
key = if kwargs.key?(:unique_by) || kwargs.key?("unique_by")
|
63
|
-
unique_by = [kwargs[:unique_by], kwargs["unique_by"]].compact.first
|
64
|
-
IdempotencyKey.generate(unique_by: unique_by, job_class: name)
|
65
|
-
else
|
66
|
-
IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
67
|
-
end
|
62
|
+
key = IdempotencyKey.new(acidic_identifier).value_for(serialized_job)
|
68
63
|
|
69
64
|
AcidicJob::Run.create!(
|
70
65
|
staged: true,
|
@@ -2,21 +2,22 @@
|
|
2
2
|
|
3
3
|
module AcidicJob
|
4
4
|
class IdempotencyKey
|
5
|
-
def self.generate(unique_by:, job_class:)
|
6
|
-
new(:job_args).value_for({ "job_class" => job_class }, Marshal.dump(unique_by))
|
7
|
-
end
|
8
|
-
|
9
5
|
def initialize(identifier = :job_id)
|
10
6
|
@identifier = identifier
|
11
7
|
end
|
12
8
|
|
13
9
|
def value_for(hash_or_job, *args, **kwargs)
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
10
|
+
value = case @identifier
|
11
|
+
when Proc
|
12
|
+
value_from_proc(hash_or_job, *args, **kwargs)
|
13
|
+
when :job_args
|
14
|
+
value_from_job_args(hash_or_job, *args, **kwargs)
|
18
15
|
else
|
19
|
-
|
16
|
+
if hash_or_job.is_a?(Hash)
|
17
|
+
value_from_job_id_for_hash(hash_or_job)
|
18
|
+
else
|
19
|
+
value_from_job_id_for_obj(hash_or_job)
|
20
|
+
end
|
20
21
|
end
|
21
22
|
|
22
23
|
value || value_from_job_args(hash_or_job, *args, **kwargs)
|
@@ -62,5 +63,12 @@ module AcidicJob
|
|
62
63
|
|
63
64
|
Digest::SHA1.hexdigest [worker_class, args, kwargs].flatten.join
|
64
65
|
end
|
66
|
+
|
67
|
+
def value_from_proc(_hash_or_job, *args, **kwargs)
|
68
|
+
return if args.empty? && kwargs.empty?
|
69
|
+
|
70
|
+
idempotency_args = Array(@identifier.call(*args, **kwargs))
|
71
|
+
Digest::SHA1.hexdigest idempotency_args.flatten.join
|
72
|
+
end
|
65
73
|
end
|
66
74
|
end
|
@@ -4,46 +4,19 @@ module AcidicJob
|
|
4
4
|
# NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
|
5
5
|
module PerformWrapper
|
6
6
|
def perform(*args, **kwargs)
|
7
|
-
|
7
|
+
@__acidic_job_args = args
|
8
|
+
@__acidic_job_kwargs = kwargs
|
8
9
|
|
9
10
|
# we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
|
10
|
-
if
|
11
|
-
|
12
|
-
elsif
|
13
|
-
|
11
|
+
if defined?(ActiveJob) && self.class < ActiveJob::Base
|
12
|
+
super(*args, **kwargs)
|
13
|
+
elsif defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
|
14
|
+
run_callbacks :perform do
|
15
|
+
super(*args, **kwargs)
|
16
|
+
end
|
14
17
|
else
|
15
18
|
raise UnknownJobAdapter
|
16
19
|
end
|
17
20
|
end
|
18
|
-
|
19
|
-
def sk_job?
|
20
|
-
defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
|
21
|
-
end
|
22
|
-
|
23
|
-
def aj_job?
|
24
|
-
defined?(ActiveJob) && self.class < ActiveJob::Base
|
25
|
-
end
|
26
|
-
|
27
|
-
private
|
28
|
-
|
29
|
-
# don't run `perform` callbacks, as ActiveJob already does this
|
30
|
-
def __acidic_job_perform_for_aj(super_method, *args, **kwargs)
|
31
|
-
__acidic_job_perform_base(super_method, *args, **kwargs)
|
32
|
-
end
|
33
|
-
|
34
|
-
# ensure to run `perform` callbacks
|
35
|
-
def __acidic_job_perform_for_sk(super_method, *args, **kwargs)
|
36
|
-
run_callbacks :perform do
|
37
|
-
__acidic_job_perform_base(super_method, *args, **kwargs)
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
|
-
# capture arguments passed to `perform` to be used by AcidicJob later
|
42
|
-
def __acidic_job_perform_base(super_method, *args, **kwargs)
|
43
|
-
@__acidic_job_args = args
|
44
|
-
@__acidic_job_kwargs = kwargs
|
45
|
-
|
46
|
-
super_method.call(*args, **kwargs)
|
47
|
-
end
|
48
21
|
end
|
49
22
|
end
|
data/lib/acidic_job/staging.rb
CHANGED
data/lib/acidic_job/version.rb
CHANGED
data/lib/acidic_job.rb
CHANGED
@@ -50,6 +50,7 @@ module AcidicJob
|
|
50
50
|
klass.instance_variable_set(:@acidic_identifier, :job_id)
|
51
51
|
klass.define_singleton_method(:acidic_by_job_id) { @acidic_identifier = :job_id }
|
52
52
|
klass.define_singleton_method(:acidic_by_job_args) { @acidic_identifier = :job_args }
|
53
|
+
klass.define_singleton_method(:acidic_by) { |proc| @acidic_identifier = proc }
|
53
54
|
end
|
54
55
|
|
55
56
|
included do
|
@@ -67,9 +68,20 @@ module AcidicJob
|
|
67
68
|
end
|
68
69
|
end
|
69
70
|
|
70
|
-
def
|
71
|
+
def initialize(*args, **kwargs)
|
71
72
|
# ensure this instance variable is always defined
|
72
73
|
@__acidic_job_steps = []
|
74
|
+
@__acidic_job_args = args
|
75
|
+
@__acidic_job_kwargs = kwargs
|
76
|
+
|
77
|
+
if method(__method__).super_method.arity.zero?
|
78
|
+
super()
|
79
|
+
else
|
80
|
+
super(*args, **kwargs)
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def with_acidity(providing: {})
|
73
85
|
# execute the block to gather the info on what steps are defined for this job workflow
|
74
86
|
yield
|
75
87
|
|
@@ -98,7 +110,6 @@ module AcidicJob
|
|
98
110
|
FinishedPoint.new
|
99
111
|
end
|
100
112
|
|
101
|
-
# TODO: allow idempotency to be defined by args OR job id
|
102
113
|
# rubocop:disable Naming/MemoizedInstanceVariableName
|
103
114
|
def idempotency_key
|
104
115
|
if defined?(@__acidic_job_idempotency_key) && !@__acidic_job_idempotency_key.nil?
|
@@ -107,7 +118,7 @@ module AcidicJob
|
|
107
118
|
|
108
119
|
acidic_identifier = self.class.acidic_identifier
|
109
120
|
@__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
|
110
|
-
.value_for(self,
|
121
|
+
.value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
|
111
122
|
end
|
112
123
|
# rubocop:enable Naming/MemoizedInstanceVariableName
|
113
124
|
|
@@ -124,7 +135,7 @@ module AcidicJob
|
|
124
135
|
|
125
136
|
# if any step calls `safely_finish_acidic_job` or the workflow has simply completed,
|
126
137
|
# be sure to break out of the loop
|
127
|
-
if recovery_point == Run::FINISHED_RECOVERY_POINT.to_s # rubocop:disable Style/GuardClause
|
138
|
+
if recovery_point.to_s == Run::FINISHED_RECOVERY_POINT.to_s # rubocop:disable Style/GuardClause
|
128
139
|
break
|
129
140
|
elsif current_step.nil?
|
130
141
|
raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
|
@@ -191,7 +202,7 @@ module AcidicJob
|
|
191
202
|
|
192
203
|
ActiveRecord::Base.transaction(isolation: isolation_level) do
|
193
204
|
run = Run.find_by(idempotency_key: idempotency_key)
|
194
|
-
serialized_job = serialize_job(
|
205
|
+
serialized_job = serialize_job(*@__acidic_job_args, **@__acidic_job_kwargs)
|
195
206
|
|
196
207
|
if run.present?
|
197
208
|
# Programs enqueuing multiple jobs with different parameters but the
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: acidic_job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.pre14
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fractaledmind
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-03-
|
11
|
+
date: 2022-03-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -63,6 +63,7 @@ files:
|
|
63
63
|
- ".gitignore"
|
64
64
|
- ".rubocop.yml"
|
65
65
|
- ".ruby_version"
|
66
|
+
- ".tool-versions"
|
66
67
|
- Gemfile
|
67
68
|
- Gemfile.lock
|
68
69
|
- LICENSE
|