acidic_job 1.0.0.beta.1 → 1.0.0.beta.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +3 -2
- data/.gitignore +1 -1
- data/Gemfile.lock +4 -10
- data/README.md +101 -102
- data/acidic_job.gemspec +0 -2
- data/gemfiles/{rails_6.1.gemfile → rails_6.1_sidekiq_6.4.gemfile} +2 -0
- data/gemfiles/rails_6.1_sidekiq_6.5.gemfile +10 -0
- data/gemfiles/{rails_7.0.gemfile → rails_7.0_sidekiq_6.4.gemfile} +2 -0
- data/gemfiles/rails_7.0_sidekiq_6.5.gemfile +10 -0
- data/gemfiles/rails_7.1_sidekiq_6.4.gemfile +10 -0
- data/gemfiles/rails_7.1_sidekiq_6.5.gemfile +10 -0
- data/lib/acidic_job/active_kiq.rb +2 -2
- data/lib/acidic_job/extensions/action_mailer.rb +19 -0
- data/lib/acidic_job/extensions/noticed.rb +46 -0
- data/lib/acidic_job/mixin.rb +6 -0
- data/lib/acidic_job/perform_wrapper.rb +3 -3
- data/lib/acidic_job/run.rb +1 -1
- data/lib/acidic_job/testing.rb +73 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +2 -0
- metadata +11 -33
- data/bin/sandbox +0 -1958
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d0e5e42007b9069a560d15f86869a6ba8efb9a45866b39d1ba761917f81188ba
|
4
|
+
data.tar.gz: bdfe50d3598ed4466e9c9b5033e53619e0a4c43fb7dbc3c8b869876ad9d57f40
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 17c7a98093f52614d0c071ef947f2fe6ea361843d3f0d494451b4f77966e3d67d1a53ef37ad75bc229a97ccb898a3cb2d193642f5bae906cd333181e34177f48
|
7
|
+
data.tar.gz: be58b523f666a0d0bec5adef39929441cc1cedc7e0b9f2d5f6cb3ba541284b4ba6b33c76e5a03afa6129fd0ac4658753d2b8d2a210b7be5b1c9847e98478c10e
|
data/.github/workflows/main.yml
CHANGED
@@ -10,7 +10,8 @@ jobs:
|
|
10
10
|
fail-fast: false
|
11
11
|
matrix:
|
12
12
|
ruby: ["2.7", "3.0", "3.1"]
|
13
|
-
rails: [
|
13
|
+
rails: ["6.1", "7.0"]
|
14
|
+
sidekiq: ["6.4", "6.5"]
|
14
15
|
services:
|
15
16
|
redis:
|
16
17
|
image: redis
|
@@ -23,7 +24,7 @@ jobs:
|
|
23
24
|
- 6379:6379
|
24
25
|
|
25
26
|
env:
|
26
|
-
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/rails_${{ matrix.rails }}.gemfile
|
27
|
+
BUNDLE_GEMFILE: ${{ github.workspace }}/gemfiles/rails_${{ matrix.rails }}_sidekiq_${{ matrix.sidekiq }}.gemfile
|
27
28
|
|
28
29
|
steps:
|
29
30
|
- uses: actions/checkout@v2
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
acidic_job (1.0.0.beta.
|
4
|
+
acidic_job (1.0.0.beta.2)
|
5
5
|
activejob
|
6
6
|
activerecord
|
7
7
|
activesupport
|
@@ -146,8 +146,6 @@ GEM
|
|
146
146
|
parallel (1.22.1)
|
147
147
|
parser (3.1.2.0)
|
148
148
|
ast (~> 2.4.1)
|
149
|
-
psych (4.0.4)
|
150
|
-
stringio
|
151
149
|
public_suffix (4.0.7)
|
152
150
|
racc (1.6.0)
|
153
151
|
rack (2.2.4)
|
@@ -201,10 +199,10 @@ GEM
|
|
201
199
|
rubocop-rake (0.6.0)
|
202
200
|
rubocop (~> 1.0)
|
203
201
|
ruby-progressbar (1.11.0)
|
204
|
-
sidekiq (6.5.
|
202
|
+
sidekiq (6.5.3)
|
205
203
|
connection_pool (>= 2.2.2)
|
206
204
|
rack (~> 2.0)
|
207
|
-
redis (>= 4.
|
205
|
+
redis (>= 4.5.0)
|
208
206
|
simplecov (0.21.2)
|
209
207
|
docile (~> 1.1)
|
210
208
|
simplecov-html (~> 0.11)
|
@@ -212,7 +210,6 @@ GEM
|
|
212
210
|
simplecov-html (0.12.3)
|
213
211
|
simplecov_json_formatter (0.1.4)
|
214
212
|
sqlite3 (1.4.4)
|
215
|
-
stringio (3.0.2)
|
216
213
|
strscan (3.0.3)
|
217
214
|
thor (1.2.1)
|
218
215
|
timeout (0.3.0)
|
@@ -222,7 +219,6 @@ GEM
|
|
222
219
|
unf_ext
|
223
220
|
unf_ext (0.0.8.2)
|
224
221
|
unicode-display_width (2.2.0)
|
225
|
-
warning (1.3.0)
|
226
222
|
websocket-driver (0.7.5)
|
227
223
|
websocket-extensions (>= 0.1.0)
|
228
224
|
websocket-extensions (0.1.5)
|
@@ -238,7 +234,6 @@ DEPENDENCIES
|
|
238
234
|
minitest
|
239
235
|
net-smtp
|
240
236
|
noticed
|
241
|
-
psych (> 4.0)
|
242
237
|
railties
|
243
238
|
rake
|
244
239
|
rubocop
|
@@ -247,7 +242,6 @@ DEPENDENCIES
|
|
247
242
|
sidekiq
|
248
243
|
simplecov
|
249
244
|
sqlite3
|
250
|
-
warning
|
251
245
|
|
252
246
|
BUNDLED WITH
|
253
|
-
2.
|
247
|
+
2.3.19
|
data/README.md
CHANGED
@@ -3,9 +3,9 @@
|
|
3
3
|
[![Gem Version](https://badge.fury.io/rb/acidic_job.svg)](https://badge.fury.io/rb/acidic_job)
|
4
4
|
![main workflow](https://github.com/fractaledmind/acidic_job/actions/workflows/main.yml/badge.svg)
|
5
5
|
|
6
|
-
|
6
|
+
## Idempotent operations for Rails apps (for ActiveJob or Sidekiq)
|
7
7
|
|
8
|
-
At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`, or we Rubyists can use the tried and true power of pure `Sidekiq`. With either we can easily trigger from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed seamlessly.
|
8
|
+
At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`, or we Rubyists can use the tried and true power of pure `Sidekiq`. With either we can easily trigger operations from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed seamlessly.
|
9
9
|
|
10
10
|
However, in order to ensure that our operational jobs are _robust_, we need to ensure that they are properly [idempotent and transactional](https://github.com/mperham/sidekiq/wiki/Best-Practices#2-make-your-job-idempotent-and-transactional). As stated in the [GitLab Sidekiq Style Guide](https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#idempotent-jobs):
|
11
11
|
|
@@ -15,14 +15,33 @@ However, in order to ensure that our operational jobs are _robust_, we need to e
|
|
15
15
|
|
16
16
|
This is, of course, far easier said than done. Thus, `AcidicJob`.
|
17
17
|
|
18
|
-
`AcidicJob` provides a framework to help you make your operational jobs atomic ⚛️, consistent 🤖, isolated 🕴🏼, and durable ⛰️. Its conceptual framework is directly inspired by a truly wonderful loosely collected series of articles written by Brandur Leach, which together lay out core techniques and principles required to make an HTTP API properly ACIDic:
|
18
|
+
`AcidicJob` provides a framework to help you make your operational jobs atomic ⚛️, consistent 🤖, isolated 🕴🏼, and durable ⛰️. Its conceptual framework is directly inspired by a truly wonderful loosely collected series of articles written by [Brandur Leach](https://twitter.com/brandur), which together lay out core techniques and principles required to make an HTTP API properly ACIDic:
|
19
19
|
|
20
|
-
1. https://brandur.org/acid
|
21
|
-
2. https://brandur.org/http-transactions
|
22
|
-
3. https://brandur.org/job-drain
|
23
|
-
4. https://brandur.org/idempotency-keys
|
20
|
+
1. [Building Robust Systems with ACID and Constraints](https://brandur.org/acid)
|
21
|
+
2. [Using Atomic Transactions to Power an Idempotent API](https://brandur.org/http-transactions)
|
22
|
+
3. [Transactionally Staged Job Drains in Postgres](https://brandur.org/job-drain)
|
23
|
+
4. [Implementing Stripe-like Idempotency Keys in Postgres](https://brandur.org/idempotency-keys)
|
24
24
|
|
25
|
-
`AcidicJob` brings these techniques and principles into the world of a standard Rails application.
|
25
|
+
Seriously, go and read these articles. `AcidicJob` brings these techniques and principles into the world of a standard Rails application, treating your background jobs like an internal API of sorts. It provides a suite of functionality that empowers you to create complex, robust, and _acidic_ jobs.
|
26
|
+
|
27
|
+
## Key Features
|
28
|
+
|
29
|
+
* **Transactional Steps**
|
30
|
+
break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
|
31
|
+
* **Steps that Await Jobs**
|
32
|
+
have workflow steps await other jobs, which will be enqueued and processed independently, and only when they all have finished will the parent job be re-enqueued to continue the workflow
|
33
|
+
* **Iterable Steps**
|
34
|
+
define steps that iterate over some collection fully until moving on to the next step
|
35
|
+
* **Persisted Attributes**
|
36
|
+
when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
|
37
|
+
* **Transactionally Staged Jobs**
|
38
|
+
enqueue additional jobs within the acidic transaction safely
|
39
|
+
* **Custom Idempotency Keys**
|
40
|
+
use something other than the job ID for the idempotency key of the job run
|
41
|
+
* **Sidekiq Callbacks**
|
42
|
+
bring ActiveJob-like callbacks into your pure Sidekiq Workers
|
43
|
+
* **Run Finished Callbacks**
|
44
|
+
set callbacks for when a job run finishes fully
|
26
45
|
|
27
46
|
## Installation
|
28
47
|
|
@@ -48,59 +67,45 @@ rails generate acidic_job:install
|
|
48
67
|
|
49
68
|
## Usage
|
50
69
|
|
51
|
-
`AcidicJob`
|
70
|
+
`AcidicJob` brings the most seamless experience when you inject it into every job in your application. This can be done most easily by simply having your `ApplicationJob` inherit from `AcidicJob::Base` (if using `ActiveJob`; inherit from `AcidicJob::ActiveKiq` if using pure Sidekiq workers):
|
52
71
|
|
53
72
|
```ruby
|
54
|
-
class ApplicationJob <
|
55
|
-
include AcidicJob
|
73
|
+
class ApplicationJob < AcidicJob::Base
|
56
74
|
end
|
57
75
|
```
|
58
76
|
|
59
|
-
This is useful because the module needs to be mixed into any and all jobs that you want to either make acidic or enqueue acidicly.
|
77
|
+
This is useful because the module needs to be mixed into any and all jobs that you want to either [1] make acidic or [2] enqueue acidicly.
|
60
78
|
|
61
|
-
|
79
|
+
If you only want to inject `AcidicJob` into a single job, you can include our concern `AcidicJob::Mixin` instead:
|
62
80
|
|
63
|
-
|
64
|
-
|
65
|
-
|
81
|
+
```ruby
|
82
|
+
class SomeJob < ApplicationJob
|
83
|
+
include AcidicJob::Mixin
|
84
|
+
end
|
85
|
+
```
|
66
86
|
|
67
|
-
|
68
|
-
break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
|
69
|
-
* **Steps that Await Jobs**
|
70
|
-
have workflow steps await other jobs, which will be enqueued and processed independently, and only when they all have finished will the parent job be re-enqueued to continue the workflow
|
71
|
-
* **Iterable Steps**
|
72
|
-
define steps that iterate over some collection fully until moving on to the next step
|
73
|
-
* **Persisted Attributes**
|
74
|
-
when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
|
75
|
-
* **Transactionally Staged Jobs**
|
76
|
-
enqueue additional jobs within the acidic transaction safely
|
77
|
-
* **Custom Idempotency Keys**
|
78
|
-
use something other than the job ID for the idempotency key of the job run
|
79
|
-
* **Sidekiq Callbacks**
|
80
|
-
bring ActiveJob-like callbacks into your pure Sidekiq Workers
|
81
|
-
* **Run Finished Callbacks**
|
82
|
-
set callbacks for when a job run finishes fully
|
87
|
+
## Key Features (in depth)
|
83
88
|
|
84
89
|
|
85
90
|
### Transactional Steps
|
86
91
|
|
87
|
-
The first and foundational feature `acidic_job` provides is the `
|
92
|
+
The first and foundational feature `acidic_job` provides is the `with_acidic_workflow` method, which takes a block of transactional step methods (defined via the `step`) method:
|
88
93
|
|
89
94
|
```ruby
|
90
|
-
class RideCreateJob <
|
91
|
-
include AcidicJob
|
92
|
-
|
95
|
+
class RideCreateJob < AcidicJob::Base
|
93
96
|
def perform(user_id, ride_params)
|
94
97
|
@user = User.find(user_id)
|
95
98
|
@params = ride_params
|
96
99
|
|
97
|
-
|
98
|
-
step :create_ride_and_audit_record
|
99
|
-
step :create_stripe_charge
|
100
|
-
step :send_receipt
|
100
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
101
|
+
workflow.step :create_ride_and_audit_record
|
102
|
+
workflow.step :create_stripe_charge
|
103
|
+
workflow.step :send_receipt
|
101
104
|
end
|
102
105
|
end
|
103
106
|
|
107
|
+
private
|
108
|
+
|
104
109
|
def create_ride_and_audit_record
|
105
110
|
# ...
|
106
111
|
end
|
@@ -115,43 +120,39 @@ class RideCreateJob < ActiveJob::Base
|
|
115
120
|
end
|
116
121
|
```
|
117
122
|
|
118
|
-
`
|
123
|
+
`with_acidic_workflow` takes only the `persisting:` named parameter (optionally) and a block (required) where you define the steps of this operation. `step` simply takes the name of a method available in the job. That's all!
|
119
124
|
|
120
|
-
Now, each execution of this job will find or create an `AcidicJob::Run` record, which we leverage to wrap every step in a database transaction. Moreover, this database record allows
|
125
|
+
Now, each execution of this job will find or create an `AcidicJob::Run` record, which we leverage to wrap every step in a database transaction. Moreover, this database record allows us to ensure that if your job fails on step 3, when it retries, it will simply jump right back to trying to execute the method defined for the 3rd step, _**and won't even execute the first two step methods**_. This means your step methods only need to be idempotent on failure, not on success, since they will never be run again if they succeed.
|
121
126
|
|
122
127
|
|
123
128
|
### Steps that Await Jobs
|
124
129
|
|
125
|
-
By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous jobs to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different jobs enqueued on 3 different queues, each running in parallel. Once all 3 jobs succeed, `
|
130
|
+
By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous jobs to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different jobs enqueued on 3 different queues, each running in parallel. Once (and only once) all 3 jobs succeed, `AcidicJob` will re-enqueue the parent job and it will move on to step 2. That's right, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations.
|
126
131
|
|
127
132
|
```ruby
|
128
|
-
class RideCreateJob <
|
129
|
-
include AcidicJob
|
130
|
-
|
133
|
+
class RideCreateJob < AcidicJob::Base
|
131
134
|
def perform(user_id, ride_params)
|
132
135
|
@user = User.find(user_id)
|
133
136
|
@params = ride_params
|
134
137
|
|
135
|
-
|
136
|
-
step :create_ride_and_audit_record, awaits: [SomeJob, AnotherJob]
|
137
|
-
step :create_stripe_charge
|
138
|
-
step :send_receipt
|
138
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
139
|
+
workflow.step :create_ride_and_audit_record, awaits: [SomeJob, AnotherJob]
|
140
|
+
workflow.step :create_stripe_charge
|
141
|
+
workflow.step :send_receipt
|
139
142
|
end
|
140
143
|
end
|
141
144
|
end
|
142
145
|
```
|
143
146
|
|
144
|
-
If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `
|
147
|
+
If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `AcidicJob` will add to your jobs:
|
145
148
|
|
146
149
|
```ruby
|
147
|
-
class RideCreateJob <
|
148
|
-
include AcidicJob
|
149
|
-
|
150
|
+
class RideCreateJob < AcidicJob::Base
|
150
151
|
def perform(user_id, ride_params)
|
151
152
|
@user = User.find(user_id)
|
152
153
|
@params = ride_params
|
153
154
|
|
154
|
-
|
155
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
155
156
|
step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value'), AnotherJob.with(1, 2, 3, some: 'thing')]
|
156
157
|
step :create_stripe_charge
|
157
158
|
step :send_receipt
|
@@ -165,21 +166,20 @@ If your step awaits multiple jobs (e.g. `awaits: [SomeJob, AnotherJob.with('argu
|
|
165
166
|
In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
|
166
167
|
|
167
168
|
```ruby
|
168
|
-
class RideCreateJob <
|
169
|
-
include AcidicJob
|
170
|
-
set_callback :finish, :after, :delete_run_record
|
171
|
-
|
169
|
+
class RideCreateJob < AcidicJob::Base
|
172
170
|
def perform(user_id, ride_params)
|
173
171
|
@user = User.find(user_id)
|
174
172
|
@params = ride_params
|
175
173
|
|
176
|
-
|
174
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
177
175
|
step :create_ride_and_audit_record, awaits: :dynamic_awaits
|
178
176
|
step :create_stripe_charge
|
179
177
|
step :send_receipt
|
180
178
|
end
|
181
179
|
end
|
182
|
-
|
180
|
+
|
181
|
+
private
|
182
|
+
|
183
183
|
def dynamic_awaits
|
184
184
|
if @params["key"].present?
|
185
185
|
[SomeJob.with('argument_1', keyword: 'value')]
|
@@ -193,41 +193,41 @@ end
|
|
193
193
|
|
194
194
|
### Iterable Steps
|
195
195
|
|
196
|
-
Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `
|
196
|
+
Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to bind that method to a specific the collection, and `AcidicJob` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
|
197
197
|
|
198
198
|
```ruby
|
199
|
-
class ExampleJob <
|
200
|
-
include AcidicJob
|
201
|
-
|
199
|
+
class ExampleJob < AcidicJob::Base
|
202
200
|
def perform(record:)
|
203
|
-
|
204
|
-
step :process_item, for_each: :collection
|
205
|
-
step :next_step
|
201
|
+
with_acidic_workflow persisting: { collection: [1, 2, 3, 4, 5] } do |workflow|
|
202
|
+
workflow.step :process_item, for_each: :collection
|
203
|
+
workflow.step :next_step
|
206
204
|
end
|
207
205
|
end
|
208
|
-
|
206
|
+
|
207
|
+
private
|
208
|
+
|
209
209
|
def process_item(item)
|
210
|
-
# do whatever work needs to be done with
|
210
|
+
# do whatever work needs to be done with an individual item from `collection`
|
211
211
|
end
|
212
212
|
end
|
213
213
|
```
|
214
214
|
|
215
|
-
**Note:** This feature relies on the "Persisted Attributes" feature detailed below. This means that you can only iterate over collections that
|
215
|
+
**Note:** This feature relies on the "Persisted Attributes" feature detailed below. This means that you can only iterate over collections that ActiveJob can serialize. See [the Rails Guide on `ActiveJob`](https://edgeguides.rubyonrails.org/active_job_basics.html#supported-types-for-arguments) for more info.
|
216
216
|
|
217
217
|
|
218
218
|
### Persisted Attributes
|
219
219
|
|
220
|
-
|
220
|
+
The `persisting` option on the `with_acidic_workflow` method allows you to create a cross-step, cross-retry context. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::Run` and manually persisting getters and setters that sync with the database record.
|
221
221
|
|
222
|
-
|
223
|
-
class RideCreateJob < ActiveJob::Base
|
224
|
-
include AcidicJob
|
222
|
+
The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to mark attributes that will be set _during a step_ via `persisting`. This means, the initial value will almost always be `nil`. If you need a default initial value, however, you can always provide that value to `persisting`.
|
225
223
|
|
224
|
+
```ruby
|
225
|
+
class RideCreateJob < AcidicJob::Base
|
226
226
|
def perform(user_id, ride_params)
|
227
227
|
@user = User.find(user_id)
|
228
228
|
@params = ride_params
|
229
229
|
|
230
|
-
|
230
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
231
231
|
step :create_ride_and_audit_record
|
232
232
|
step :create_stripe_charge
|
233
233
|
step :send_receipt
|
@@ -246,28 +246,24 @@ class RideCreateJob < ActiveJob::Base
|
|
246
246
|
end
|
247
247
|
```
|
248
248
|
|
249
|
-
**Note:** This does mean that you are restricted to objects that can be serialized by **`ActiveJob`** (for more info, see [
|
249
|
+
**Note:** This does mean that you are restricted to objects that can be serialized by **`ActiveJob`** (for more info, see [the Rails Guide on `ActiveJob`](https://edgeguides.rubyonrails.org/active_job_basics.html#supported-types-for-arguments)). This means you can persist ActiveRecord models, and any simple Ruby data types, but you can't persist things like Procs or custom class instances, for example.
|
250
250
|
|
251
251
|
**Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
|
252
252
|
|
253
|
-
The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
|
254
|
-
|
255
253
|
|
256
254
|
### Transactionally Staged Jobs
|
257
255
|
|
258
256
|
A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
|
259
257
|
|
260
|
-
In order to mitigate against such issues without forcing you to use a database-backed job queue, `
|
258
|
+
In order to mitigate against such issues without forcing you to use a database-backed job queue, `AcidicJob` provides `perform_acidicly` and `deliver_acidicly` methods to "transactionally stage" enqueuing other jobs from within a step (whether another `ActiveJob` or a `Sidekiq::Worker` or an `ActionMailer` delivery). These methods will create a new `AcidicJob::Run` record, but inside of the database transaction of the `step`. Upon commit of that transaction, a model callback pushes the job to your actual job queue. Once the job has been successfully performed, the `AcidicJob::Run` record is deleted so that this table doesn't grow unbounded and unnecessarily.
|
261
259
|
|
262
260
|
```ruby
|
263
|
-
class RideCreateJob <
|
264
|
-
include AcidicJob
|
265
|
-
|
261
|
+
class RideCreateJob < AcidicJob::Base
|
266
262
|
def perform(user_id, ride_params)
|
267
263
|
@user = User.find(user_id)
|
268
264
|
@params = ride_params
|
269
265
|
|
270
|
-
|
266
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
271
267
|
step :create_ride_and_audit_record
|
272
268
|
step :create_stripe_charge
|
273
269
|
step :send_receipt
|
@@ -290,8 +286,7 @@ By default, `AcidicJob` uses the job identifier provided by the queueing system
|
|
290
286
|
Firstly, you can configure your job class to explicitly use either the job identifier or the job arguments as the foundation for the idempotency key. A job class that calls the `acidic_by_job_id` class method (which is the default behavior) will simply make the job run's idempotency key the job's identifier:
|
291
287
|
|
292
288
|
```ruby
|
293
|
-
class ExampleJob <
|
294
|
-
include AcidicJob
|
289
|
+
class ExampleJob < AcidicJob::Base
|
295
290
|
acidic_by_job_id
|
296
291
|
|
297
292
|
def perform
|
@@ -302,8 +297,7 @@ end
|
|
302
297
|
Conversely, a job class can use the `acidic_by_job_args` method to configure that job class to use the arguments passed to the job as the foundation for the job run's idempotency key:
|
303
298
|
|
304
299
|
```ruby
|
305
|
-
class ExampleJob <
|
306
|
-
include AcidicJob
|
300
|
+
class ExampleJob < AcidicJob::Base
|
307
301
|
acidic_by_job_args
|
308
302
|
|
309
303
|
def perform(arg_1, arg_2)
|
@@ -315,48 +309,53 @@ end
|
|
315
309
|
These options cover the two common situations, but sometimes our systems need finer-grained control. For example, our job might take some record as the job argument, but we need to use a combination of the record identifier and record status as the foundation for the idempotency key. In these cases you can pass a `Proc` to an `acidic_by` class method:
|
316
310
|
|
317
311
|
```ruby
|
318
|
-
class ExampleJob <
|
319
|
-
|
320
|
-
acidic_by ->(record:) { [record.id, record.status] }
|
312
|
+
class ExampleJob < AcidicJob::Base
|
313
|
+
acidic_by -> { [@record.id, @record.status] }
|
321
314
|
|
322
315
|
def perform(record:)
|
323
|
-
|
316
|
+
@record = record
|
317
|
+
|
318
|
+
# the idempotency key will be based on whatever the values of `@record.id` and `@record.status` are
|
319
|
+
with_acidic_workflow do |workflow|
|
320
|
+
workflow.step :do_something
|
321
|
+
end
|
324
322
|
end
|
325
323
|
end
|
326
324
|
```
|
327
325
|
|
328
|
-
> **Note:** The
|
326
|
+
> **Note:** The `acidic_by` proc _will be executed in the context of the job instance_ at the moment the `with_acidic_workflow` method is called. This means it will have access to any instance variables defined in your `perform` method up to that point.
|
329
327
|
|
330
328
|
|
331
329
|
### Sidekiq Callbacks
|
332
330
|
|
333
|
-
In order to ensure that `AcidicJob::
|
331
|
+
In order to ensure that staged `AcidicJob::Run` records are only destroyed once the related job has been successfully performed, whether it is an ActiveJob or a Sidekiq Worker, `AcidicJob` also extends Sidekiq to support the [ActiveJob callback interface](https://edgeguides.rubyonrails.org/active_job_basics.html#callbacks).
|
334
332
|
|
335
|
-
This allows
|
333
|
+
This allows us to use an `after_perform` callback to delete the `AcidicJob::Run` record, whether you are using the gem with ActiveJob or pure Sidekiq Workers. Of course, this means that you can add your own callbacks to any jobs or workers that include the `AcidicJob` module as well.
|
336
334
|
|
337
335
|
|
338
336
|
### Run Finished Callbacks
|
339
337
|
|
340
|
-
When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `
|
338
|
+
When working with workflow jobs that make use of the `awaits` feature for a step, it is important to remember that the `after_perform` callback will be called _as soon as the first `awaits` step has enqueued job_, and **not** when the entire job run has finished. `AcidicJob` allows the `perform` method to finish so that the queue for the workflow job is cleared to pick up new work while the `awaits` jobs are running. `AcidicJob` will automatically re-enqueue the workflow job and progress to the next step when all of the `awaits` jobs have successfully finished. However, this means that `after_perform` **is not necessarily** the same as `after_finish`. In order to provide the opportunity for you to execute callback logic _if and only if_ a job run has finished, we provide callback hooks for the `finish` event.
|
341
339
|
|
342
340
|
For example, you could use this hook to immediately clean up the `AcidicJob::Run` database record whenever the workflow job finishes successfully like so:
|
343
341
|
|
344
342
|
```ruby
|
345
|
-
class RideCreateJob <
|
346
|
-
include AcidicJob
|
343
|
+
class RideCreateJob < AcidicJob::Base
|
347
344
|
set_callback :finish, :after, :delete_run_record
|
348
345
|
|
349
346
|
def perform(user_id, ride_params)
|
350
347
|
@user = User.find(user_id)
|
351
348
|
@params = ride_params
|
352
349
|
|
353
|
-
|
350
|
+
with_acidic_workflow persisting: { ride: nil } do |workflow|
|
354
351
|
step :create_ride_and_audit_record, awaits: [SomeJob.with('argument_1', keyword: 'value')]
|
355
352
|
step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
|
356
353
|
step :send_receipt
|
357
354
|
end
|
358
355
|
end
|
359
|
-
|
356
|
+
|
357
|
+
private
|
358
|
+
|
360
359
|
def delete_run_record
|
361
360
|
return unless acidic_job_run.succeeded?
|
362
361
|
|
@@ -374,11 +373,11 @@ When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionI
|
|
374
373
|
ActiveRecord::TransactionIsolationError: cannot set transaction isolation in a nested transaction
|
375
374
|
```
|
376
375
|
|
377
|
-
This error is thrown because by default RSpec and most MiniTest test suites use database transactions to keep the test database clean between tests. The database transaction that is wrapping all of the code executed in your test is run at the standard isolation level, but
|
376
|
+
This error is thrown because by default RSpec and most MiniTest test suites use database transactions to keep the test database clean between tests. The database transaction that is wrapping all of the code executed in your test is run at the standard isolation level, but `AcidicJob` then tries to create another transaction at a more conservative isolation level. You cannot have a nested transaction that runs at a different isolation level, thus, this error.
|
378
377
|
|
379
378
|
In order to avoid this error, you need to ensure firstly that your tests that run your acidic jobs are not using a database transaction and secondly that they use some different strategy to keep your test database clean. The [DatabaseCleaner](https://github.com/DatabaseCleaner/database_cleaner) gem is a commonly used tool to manage different strategies for keeping your test database clean. As for which strategy to use, `truncation` and `deletion` are both safe, but their speed varies based on our app's table structure (see https://github.com/DatabaseCleaner/database_cleaner#what-strategy-is-fastest). Either is fine; use whichever is faster for your app.
|
380
379
|
|
381
|
-
In order to make this test setup simpler, `AcidicJob` provides a `
|
380
|
+
In order to make this test setup simpler, `AcidicJob` provides a `Testing` module that your job tests can include. It is simple; it sets `use_transactional_tests` to `false` (if the test is an `ActiveJob::TestCase`), and ensures a transaction-safe `DatabaseCleaner` strategy is run for each of your tests. Moreover, it ensures that the system's original DatabaseCleaner configuration is maintained, options included, except that any `transaction` strategies for any ORMs are replaced with a `deletion` strategy. It does so by storing whatever the system DatabaseCleaner configuration is at the start of `before_setup` phase in an instance variable and then restores that configuration at the end of `after_teardown` phase. In between, it runs the configuration thru a pipeline that selectively replaces any `transaction` strategies with a corresponding `deletion` strategy, leaving any other configured strategies untouched.
|
382
381
|
|
383
382
|
For those of you using RSpec, you can require the `acidic_job/rspec_configuration` file, which will configure RSpec in the exact same way I have used in my RSpec projects to allow me to test acidic jobs with either the `deletion` strategy but still have all of my other tests use the fast `transaction` strategy:
|
384
383
|
|
data/acidic_job.gemspec
CHANGED
@@ -34,7 +34,6 @@ Gem::Specification.new do |spec|
|
|
34
34
|
spec.add_development_dependency "minitest"
|
35
35
|
spec.add_development_dependency "net-smtp"
|
36
36
|
spec.add_development_dependency "noticed"
|
37
|
-
spec.add_development_dependency "psych", "> 4.0"
|
38
37
|
spec.add_development_dependency "railties"
|
39
38
|
spec.add_development_dependency "rake"
|
40
39
|
spec.add_development_dependency "rubocop"
|
@@ -43,7 +42,6 @@ Gem::Specification.new do |spec|
|
|
43
42
|
spec.add_development_dependency "sidekiq"
|
44
43
|
spec.add_development_dependency "simplecov"
|
45
44
|
spec.add_development_dependency "sqlite3"
|
46
|
-
spec.add_development_dependency "warning"
|
47
45
|
|
48
46
|
# For more information and examples about making a new gem, checkout our
|
49
47
|
# guide at: https://bundler.io/guides/creating_gem.html
|
@@ -26,8 +26,8 @@ module AcidicJob
|
|
26
26
|
# +opts+ are any options to configure the job
|
27
27
|
def initialize(*arguments)
|
28
28
|
@arguments = arguments
|
29
|
-
@job_id = SecureRandom.uuid
|
30
|
-
@sidekiq_options = sidekiq_options_hash || Sidekiq.default_job_options
|
29
|
+
@job_id = ::SecureRandom.uuid
|
30
|
+
@sidekiq_options = sidekiq_options_hash || ::Sidekiq.default_job_options
|
31
31
|
@queue_name = @sidekiq_options["queue"]
|
32
32
|
end
|
33
33
|
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Extensions
|
7
|
+
module ActionMailer
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
def deliver_acidicly(_options = {})
|
11
|
+
job_class = ::ActionMailer::MailDeliveryJob
|
12
|
+
job_args = [@mailer_class.name, @action.to_s, "deliver_now", @params, *@args]
|
13
|
+
job = job_class.new(job_args)
|
14
|
+
|
15
|
+
AcidicJob::Run.stage!(job)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|