acidic_job 1.0.0.pre25 → 1.0.0.pre28
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/Gemfile.lock +84 -78
- data/README.md +122 -105
- data/acidic_job.gemspec +1 -0
- data/combustion/log/test.log +0 -0
- data/lib/acidic_job/awaiting.rb +1 -1
- data/lib/acidic_job/extensions/sidekiq.rb +1 -1
- data/lib/acidic_job/run.rb +6 -5
- data/lib/acidic_job/serializer.rb +163 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +1 -1
- metadata +18 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ec26e6c8087cb17255d82e8f4b1df56d2805ae936836312135552e38c140d4e2
|
4
|
+
data.tar.gz: ec268b9d40c12aa9fd2c32d7c48bb75fa462ce2bdde4f0a27bacdb2fac853aa9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f8fba84289d1d29b00c44fe535d66c5af11e0c1addb0f7d061b760bc6a2bbd0bc79d84b7294d5bfde340990e1a49a723d4910eaa70f2c60f96eb5784b25c5f4e
|
7
|
+
data.tar.gz: 88fb08f4e42fa0bb99622b32a57c4ca3bbdb657f6d8c6cbf13b9ef64fc52307f27777e88c32102043b0b5e2f4f7f12ed066b46c69ce5cf92f3fae0948faaf648
|
data/.gitignore
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
acidic_job (1.0.0.
|
4
|
+
acidic_job (1.0.0.pre28)
|
5
5
|
activerecord
|
6
6
|
activesupport
|
7
7
|
database_cleaner
|
@@ -9,67 +9,67 @@ PATH
|
|
9
9
|
GEM
|
10
10
|
remote: https://rubygems.org/
|
11
11
|
specs:
|
12
|
-
actioncable (7.0.3)
|
13
|
-
actionpack (= 7.0.3)
|
14
|
-
activesupport (= 7.0.3)
|
12
|
+
actioncable (7.0.3.1)
|
13
|
+
actionpack (= 7.0.3.1)
|
14
|
+
activesupport (= 7.0.3.1)
|
15
15
|
nio4r (~> 2.0)
|
16
16
|
websocket-driver (>= 0.6.1)
|
17
|
-
actionmailbox (7.0.3)
|
18
|
-
actionpack (= 7.0.3)
|
19
|
-
activejob (= 7.0.3)
|
20
|
-
activerecord (= 7.0.3)
|
21
|
-
activestorage (= 7.0.3)
|
22
|
-
activesupport (= 7.0.3)
|
17
|
+
actionmailbox (7.0.3.1)
|
18
|
+
actionpack (= 7.0.3.1)
|
19
|
+
activejob (= 7.0.3.1)
|
20
|
+
activerecord (= 7.0.3.1)
|
21
|
+
activestorage (= 7.0.3.1)
|
22
|
+
activesupport (= 7.0.3.1)
|
23
23
|
mail (>= 2.7.1)
|
24
24
|
net-imap
|
25
25
|
net-pop
|
26
26
|
net-smtp
|
27
|
-
actionmailer (7.0.3)
|
28
|
-
actionpack (= 7.0.3)
|
29
|
-
actionview (= 7.0.3)
|
30
|
-
activejob (= 7.0.3)
|
31
|
-
activesupport (= 7.0.3)
|
27
|
+
actionmailer (7.0.3.1)
|
28
|
+
actionpack (= 7.0.3.1)
|
29
|
+
actionview (= 7.0.3.1)
|
30
|
+
activejob (= 7.0.3.1)
|
31
|
+
activesupport (= 7.0.3.1)
|
32
32
|
mail (~> 2.5, >= 2.5.4)
|
33
33
|
net-imap
|
34
34
|
net-pop
|
35
35
|
net-smtp
|
36
36
|
rails-dom-testing (~> 2.0)
|
37
|
-
actionpack (7.0.3)
|
38
|
-
actionview (= 7.0.3)
|
39
|
-
activesupport (= 7.0.3)
|
37
|
+
actionpack (7.0.3.1)
|
38
|
+
actionview (= 7.0.3.1)
|
39
|
+
activesupport (= 7.0.3.1)
|
40
40
|
rack (~> 2.0, >= 2.2.0)
|
41
41
|
rack-test (>= 0.6.3)
|
42
42
|
rails-dom-testing (~> 2.0)
|
43
43
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
44
|
-
actiontext (7.0.3)
|
45
|
-
actionpack (= 7.0.3)
|
46
|
-
activerecord (= 7.0.3)
|
47
|
-
activestorage (= 7.0.3)
|
48
|
-
activesupport (= 7.0.3)
|
44
|
+
actiontext (7.0.3.1)
|
45
|
+
actionpack (= 7.0.3.1)
|
46
|
+
activerecord (= 7.0.3.1)
|
47
|
+
activestorage (= 7.0.3.1)
|
48
|
+
activesupport (= 7.0.3.1)
|
49
49
|
globalid (>= 0.6.0)
|
50
50
|
nokogiri (>= 1.8.5)
|
51
|
-
actionview (7.0.3)
|
52
|
-
activesupport (= 7.0.3)
|
51
|
+
actionview (7.0.3.1)
|
52
|
+
activesupport (= 7.0.3.1)
|
53
53
|
builder (~> 3.1)
|
54
54
|
erubi (~> 1.4)
|
55
55
|
rails-dom-testing (~> 2.0)
|
56
56
|
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
57
|
-
activejob (7.0.3)
|
58
|
-
activesupport (= 7.0.3)
|
57
|
+
activejob (7.0.3.1)
|
58
|
+
activesupport (= 7.0.3.1)
|
59
59
|
globalid (>= 0.3.6)
|
60
|
-
activemodel (7.0.3)
|
61
|
-
activesupport (= 7.0.3)
|
62
|
-
activerecord (7.0.3)
|
63
|
-
activemodel (= 7.0.3)
|
64
|
-
activesupport (= 7.0.3)
|
65
|
-
activestorage (7.0.3)
|
66
|
-
actionpack (= 7.0.3)
|
67
|
-
activejob (= 7.0.3)
|
68
|
-
activerecord (= 7.0.3)
|
69
|
-
activesupport (= 7.0.3)
|
60
|
+
activemodel (7.0.3.1)
|
61
|
+
activesupport (= 7.0.3.1)
|
62
|
+
activerecord (7.0.3.1)
|
63
|
+
activemodel (= 7.0.3.1)
|
64
|
+
activesupport (= 7.0.3.1)
|
65
|
+
activestorage (7.0.3.1)
|
66
|
+
actionpack (= 7.0.3.1)
|
67
|
+
activejob (= 7.0.3.1)
|
68
|
+
activerecord (= 7.0.3.1)
|
69
|
+
activesupport (= 7.0.3.1)
|
70
70
|
marcel (~> 1.0)
|
71
71
|
mini_mime (>= 1.1.0)
|
72
|
-
activesupport (7.0.3)
|
72
|
+
activesupport (7.0.3.1)
|
73
73
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
74
74
|
i18n (>= 1.6, < 2)
|
75
75
|
minitest (>= 5.1)
|
@@ -78,7 +78,7 @@ GEM
|
|
78
78
|
public_suffix (>= 2.0.2, < 5.0)
|
79
79
|
ast (2.4.2)
|
80
80
|
builder (3.2.4)
|
81
|
-
combustion (1.3.
|
81
|
+
combustion (1.3.7)
|
82
82
|
activesupport (>= 3.0.0)
|
83
83
|
railties (>= 3.0.0)
|
84
84
|
thor (>= 0.14.6)
|
@@ -102,16 +102,17 @@ GEM
|
|
102
102
|
rake
|
103
103
|
globalid (1.0.0)
|
104
104
|
activesupport (>= 5.0)
|
105
|
-
http (5.0
|
105
|
+
http (5.1.0)
|
106
106
|
addressable (~> 2.8)
|
107
107
|
http-cookie (~> 1.0)
|
108
108
|
http-form_data (~> 2.2)
|
109
109
|
llhttp-ffi (~> 0.4.0)
|
110
|
-
http-cookie (1.0.
|
110
|
+
http-cookie (1.0.5)
|
111
111
|
domain_name (~> 0.5)
|
112
112
|
http-form_data (2.3.0)
|
113
|
-
i18n (1.
|
113
|
+
i18n (1.12.0)
|
114
114
|
concurrent-ruby (~> 1.0)
|
115
|
+
json (2.6.2)
|
115
116
|
llhttp-ffi (0.4.0)
|
116
117
|
ffi-compiler (~> 1.0)
|
117
118
|
rake (~> 13.0)
|
@@ -124,7 +125,7 @@ GEM
|
|
124
125
|
method_source (1.0.0)
|
125
126
|
mini_mime (1.1.2)
|
126
127
|
mini_portile2 (2.8.0)
|
127
|
-
minitest (5.
|
128
|
+
minitest (5.16.2)
|
128
129
|
net-imap (0.2.3)
|
129
130
|
digest
|
130
131
|
net-protocol
|
@@ -140,10 +141,10 @@ GEM
|
|
140
141
|
net-protocol
|
141
142
|
timeout
|
142
143
|
nio4r (2.5.8)
|
143
|
-
nokogiri (1.13.
|
144
|
+
nokogiri (1.13.7)
|
144
145
|
mini_portile2 (~> 2.8.0)
|
145
146
|
racc (~> 1.4)
|
146
|
-
nokogiri (1.13.
|
147
|
+
nokogiri (1.13.7-x86_64-darwin)
|
147
148
|
racc (~> 1.4)
|
148
149
|
noticed (1.5.9)
|
149
150
|
http (>= 4.0.0)
|
@@ -151,59 +152,62 @@ GEM
|
|
151
152
|
parallel (1.22.1)
|
152
153
|
parser (3.1.2.0)
|
153
154
|
ast (~> 2.4.1)
|
155
|
+
psych (4.0.4)
|
156
|
+
stringio
|
154
157
|
public_suffix (4.0.7)
|
155
158
|
racc (1.6.0)
|
156
|
-
rack (2.2.
|
157
|
-
rack-test (
|
158
|
-
rack (>= 1.
|
159
|
-
rails (7.0.3)
|
160
|
-
actioncable (= 7.0.3)
|
161
|
-
actionmailbox (= 7.0.3)
|
162
|
-
actionmailer (= 7.0.3)
|
163
|
-
actionpack (= 7.0.3)
|
164
|
-
actiontext (= 7.0.3)
|
165
|
-
actionview (= 7.0.3)
|
166
|
-
activejob (= 7.0.3)
|
167
|
-
activemodel (= 7.0.3)
|
168
|
-
activerecord (= 7.0.3)
|
169
|
-
activestorage (= 7.0.3)
|
170
|
-
activesupport (= 7.0.3)
|
159
|
+
rack (2.2.4)
|
160
|
+
rack-test (2.0.2)
|
161
|
+
rack (>= 1.3)
|
162
|
+
rails (7.0.3.1)
|
163
|
+
actioncable (= 7.0.3.1)
|
164
|
+
actionmailbox (= 7.0.3.1)
|
165
|
+
actionmailer (= 7.0.3.1)
|
166
|
+
actionpack (= 7.0.3.1)
|
167
|
+
actiontext (= 7.0.3.1)
|
168
|
+
actionview (= 7.0.3.1)
|
169
|
+
activejob (= 7.0.3.1)
|
170
|
+
activemodel (= 7.0.3.1)
|
171
|
+
activerecord (= 7.0.3.1)
|
172
|
+
activestorage (= 7.0.3.1)
|
173
|
+
activesupport (= 7.0.3.1)
|
171
174
|
bundler (>= 1.15.0)
|
172
|
-
railties (= 7.0.3)
|
175
|
+
railties (= 7.0.3.1)
|
173
176
|
rails-dom-testing (2.0.3)
|
174
177
|
activesupport (>= 4.2.0)
|
175
178
|
nokogiri (>= 1.6)
|
176
|
-
rails-html-sanitizer (1.4.
|
179
|
+
rails-html-sanitizer (1.4.3)
|
177
180
|
loofah (~> 2.3)
|
178
|
-
railties (7.0.3)
|
179
|
-
actionpack (= 7.0.3)
|
180
|
-
activesupport (= 7.0.3)
|
181
|
+
railties (7.0.3.1)
|
182
|
+
actionpack (= 7.0.3.1)
|
183
|
+
activesupport (= 7.0.3.1)
|
181
184
|
method_source
|
182
185
|
rake (>= 12.2)
|
183
186
|
thor (~> 1.0)
|
184
187
|
zeitwerk (~> 2.5)
|
185
188
|
rainbow (3.1.1)
|
186
189
|
rake (13.0.6)
|
187
|
-
redis (4.
|
188
|
-
regexp_parser (2.
|
190
|
+
redis (4.7.1)
|
191
|
+
regexp_parser (2.5.0)
|
189
192
|
rexml (3.2.5)
|
190
|
-
rubocop (1.
|
193
|
+
rubocop (1.31.2)
|
194
|
+
json (~> 2.3)
|
191
195
|
parallel (~> 1.10)
|
192
196
|
parser (>= 3.1.0.0)
|
193
197
|
rainbow (>= 2.2.2, < 4.0)
|
194
198
|
regexp_parser (>= 1.8, < 3.0)
|
195
199
|
rexml (>= 3.2.5, < 4.0)
|
196
|
-
rubocop-ast (>= 1.
|
200
|
+
rubocop-ast (>= 1.18.0, < 2.0)
|
197
201
|
ruby-progressbar (~> 1.7)
|
198
202
|
unicode-display_width (>= 1.4.0, < 3.0)
|
199
|
-
rubocop-ast (1.
|
203
|
+
rubocop-ast (1.19.1)
|
200
204
|
parser (>= 3.1.1.0)
|
201
|
-
rubocop-minitest (0.
|
205
|
+
rubocop-minitest (0.20.1)
|
202
206
|
rubocop (>= 0.90, < 2.0)
|
203
207
|
rubocop-rake (0.6.0)
|
204
208
|
rubocop (~> 1.0)
|
205
209
|
ruby-progressbar (1.11.0)
|
206
|
-
sidekiq (6.
|
210
|
+
sidekiq (6.5.1)
|
207
211
|
connection_pool (>= 2.2.2)
|
208
212
|
rack (~> 2.0)
|
209
213
|
redis (>= 4.2.0)
|
@@ -213,21 +217,22 @@ GEM
|
|
213
217
|
simplecov_json_formatter (~> 0.1)
|
214
218
|
simplecov-html (0.12.3)
|
215
219
|
simplecov_json_formatter (0.1.4)
|
216
|
-
sqlite3 (1.4.
|
220
|
+
sqlite3 (1.4.4)
|
221
|
+
stringio (3.0.2)
|
217
222
|
strscan (3.0.3)
|
218
223
|
thor (1.2.1)
|
219
|
-
timeout (0.
|
224
|
+
timeout (0.3.0)
|
220
225
|
tzinfo (2.0.4)
|
221
226
|
concurrent-ruby (~> 1.0)
|
222
227
|
unf (0.1.4)
|
223
228
|
unf_ext
|
224
|
-
unf_ext (0.0.8.
|
225
|
-
unicode-display_width (2.
|
226
|
-
warning (1.
|
229
|
+
unf_ext (0.0.8.2)
|
230
|
+
unicode-display_width (2.2.0)
|
231
|
+
warning (1.3.0)
|
227
232
|
websocket-driver (0.7.5)
|
228
233
|
websocket-extensions (>= 0.1.0)
|
229
234
|
websocket-extensions (0.1.5)
|
230
|
-
zeitwerk (2.
|
235
|
+
zeitwerk (2.6.0)
|
231
236
|
|
232
237
|
PLATFORMS
|
233
238
|
ruby
|
@@ -240,6 +245,7 @@ DEPENDENCIES
|
|
240
245
|
minitest
|
241
246
|
net-smtp
|
242
247
|
noticed
|
248
|
+
psych (> 4.0)
|
243
249
|
railties
|
244
250
|
rake
|
245
251
|
rubocop
|
data/README.md
CHANGED
@@ -64,14 +64,23 @@ It provides a suite of functionality that empowers you to create complex, robust
|
|
64
64
|
|
65
65
|
#### Key Features
|
66
66
|
|
67
|
-
* Transactional Steps
|
68
|
-
|
69
|
-
*
|
70
|
-
|
71
|
-
* Iterable Steps
|
72
|
-
|
73
|
-
*
|
74
|
-
|
67
|
+
* **Transactional Steps**
|
68
|
+
break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
|
69
|
+
* **Steps that Await Jobs**
|
70
|
+
have workflow steps await other jobs, which will be enqueued and processed independently, and only when they all have finished will the parent job be re-enqueued to continue the workflow
|
71
|
+
* **Iterable Steps**
|
72
|
+
define steps that iterate over some collection fully until moving on to the next step
|
73
|
+
* **Persisted Attributes**
|
74
|
+
when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
|
75
|
+
* **Transactionally Staged Jobs**
|
76
|
+
enqueue additional jobs within the acidic transaction safely
|
77
|
+
* **Custom Idempotency Keys**
|
78
|
+
use something other than the job ID for the idempotency key of the job run
|
79
|
+
* **Sidekiq Callbacks**
|
80
|
+
bring ActiveJob-like callbacks into your pure Sidekiq Workers
|
81
|
+
* **Run Finished Callbacks**
|
82
|
+
set callbacks for when a job run finishes fully
|
83
|
+
|
75
84
|
|
76
85
|
### Transactional Steps
|
77
86
|
|
@@ -110,15 +119,114 @@ end
|
|
110
119
|
|
111
120
|
Now, each execution of this job will find or create an `AcidicJob::Run` record, which we leverage to wrap every step in a database transaction. Moreover, this database record allows `acidic_job` to ensure that if your job fails on step 3, when it retries, it will simply jump right back to trying to execute the method defined for the 3rd step, and won't even execute the first two step methods. This means your step methods only need to be idempotent on failure, not on success, since they will never be run again if they succeed.
|
112
121
|
|
122
|
+
|
123
|
+
### Steps that Await Jobs
|
124
|
+
|
125
|
+
By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous jobs to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different jobs enqueued on 3 different queues, each running in parallel. Once all 3 jobs succeed, `acidic_job` will re-enqueue the parent job and it will move on to step 2. That's right, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations.
|
126
|
+
|
127
|
+
```ruby
|
128
|
+
class RideCreateJob < ActiveJob::Base
|
129
|
+
include AcidicJob
|
130
|
+
|
131
|
+
def perform(user_id, ride_params)
|
132
|
+
@user = User.find(user_id)
|
133
|
+
@params = ride_params
|
134
|
+
|
135
|
+
with_acidity providing: { ride: nil } do
|
136
|
+
step :create_ride_and_audit_record, awaits: [SomeJob, AnotherJob]
|
137
|
+
step :create_stripe_charge
|
138
|
+
step :send_receipt
|
139
|
+
end
|
140
|
+
end
|
141
|
+
end
|
142
|
+
```
|
143
|
+
|
144
|
+
If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
|
145
|
+
|
146
|
+
```ruby
|
147
|
+
class RideCreateJob < ActiveJob::Base
|
148
|
+
include AcidicJob
|
149
|
+
|
150
|
+
def perform(user_id, ride_params)
|
151
|
+
@user = User.find(user_id)
|
152
|
+
@params = ride_params
|
153
|
+
|
154
|
+
with_acidity providing: { ride: nil } do
|
155
|
+
step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value'), AnotherJob.with(1, 2, 3, some: 'thing')]
|
156
|
+
step :create_stripe_charge
|
157
|
+
step :send_receipt
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
```
|
162
|
+
|
163
|
+
If your step awaits multiple jobs (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`), your top level workflow job will only continue to the next step once **all** of the jobs in your `awaits` array have finished.
|
164
|
+
|
165
|
+
In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
|
166
|
+
|
167
|
+
```ruby
|
168
|
+
class RideCreateJob < ActiveJob::Base
|
169
|
+
include AcidicJob
|
170
|
+
set_callback :finish, :after, :delete_run_record
|
171
|
+
|
172
|
+
def perform(user_id, ride_params)
|
173
|
+
@user = User.find(user_id)
|
174
|
+
@params = ride_params
|
175
|
+
|
176
|
+
with_acidity providing: { ride: nil } do
|
177
|
+
step :create_ride_and_audit_record, awaits: :dynamic_awaits
|
178
|
+
step :create_stripe_charge
|
179
|
+
step :send_receipt
|
180
|
+
end
|
181
|
+
end
|
182
|
+
|
183
|
+
def dynamic_awaits
|
184
|
+
if @params["key"].present?
|
185
|
+
[SomeJob.with('argument_1', keyword: 'value')]
|
186
|
+
else
|
187
|
+
[AnotherJob.with(1, 2, 3, some: 'thing')]
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
```
|
192
|
+
|
193
|
+
|
194
|
+
### Iterable Steps
|
195
|
+
|
196
|
+
Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
|
197
|
+
|
198
|
+
```ruby
|
199
|
+
class ExampleJob < ActiveJob::Base
|
200
|
+
include AcidicJob
|
201
|
+
|
202
|
+
def perform(record:)
|
203
|
+
with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
|
204
|
+
step :process_item, for_each: :collection
|
205
|
+
step :next_step
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def process_item(item)
|
210
|
+
# do whatever work needs to be done with this individual item
|
211
|
+
end
|
212
|
+
end
|
213
|
+
```
|
214
|
+
|
215
|
+
**Note:** This feature relies on the "Persisted Attributes" feature detailed below. This means that you can only iterate over collections that ActiveRecord can serialize.
|
216
|
+
|
217
|
+
|
113
218
|
### Persisted Attributes
|
114
219
|
|
115
|
-
Any objects passed to the `providing` option on the `with_acidity` method are
|
220
|
+
Any objects passed to the `providing` option on the `with_acidity` method are made available across retries. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::Run` and manually providing getters and setters that sync with the database record.
|
116
221
|
|
117
222
|
```ruby
|
118
223
|
class RideCreateJob < ActiveJob::Base
|
119
224
|
include AcidicJob
|
120
225
|
|
121
|
-
def perform(ride_params)
|
226
|
+
def perform(user_id, ride_params)
|
227
|
+
@user = User.find(user_id)
|
228
|
+
@params = ride_params
|
229
|
+
|
122
230
|
with_acidity providing: { ride: nil } do
|
123
231
|
step :create_ride_and_audit_record
|
124
232
|
step :create_stripe_charge
|
@@ -138,12 +246,13 @@ class RideCreateJob < ActiveJob::Base
|
|
138
246
|
end
|
139
247
|
```
|
140
248
|
|
141
|
-
**Note:** This does mean that you are restricted to objects that can be serialized by ActiveRecord,
|
249
|
+
**Note:** This does mean that you are restricted to objects that can be serialized by **`ActiveJob`** (for more info, see [here](https://edgeguides.rubyonrails.org/active_job_basics.html#supported-types-for-arguments)). This means you can persist ActiveRecord models, and any simple Ruby data types, but you can't persist things like Procs or custom class instances, for example.
|
142
250
|
|
143
251
|
**Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
|
144
252
|
|
145
253
|
The default pattern you should follow when defining your `perform` method is to make any values that your `step` methods need access to, but are present at the start of the `perform` method simply instance variables. You only need to `provide` attributes that will be set _during a step_. This means, the initial value will almost always be `nil`.
|
146
254
|
|
255
|
+
|
147
256
|
### Transactionally Staged Jobs
|
148
257
|
|
149
258
|
A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
|
@@ -173,6 +282,7 @@ class RideCreateJob < ActiveJob::Base
|
|
173
282
|
end
|
174
283
|
```
|
175
284
|
|
285
|
+
|
176
286
|
### Custom Idempotency Keys
|
177
287
|
|
178
288
|
By default, `AcidicJob` uses the job identifier provided by the queueing system (ActiveJob or Sidekiq) as the idempotency key for the job run. The idempotency key is what is used to guarantee that no two runs of the same job occur. However, sometimes we need particular jobs to be idempotent based on some other criteria. In these cases, `AcidicJob` provides a collection of tools to allow you to ensure the idempotency of your jobs.
|
@@ -217,28 +327,6 @@ end
|
|
217
327
|
|
218
328
|
> **Note:** The signature of the `acidic_by` proc _needs to match the signature_ of the job's `perform` method.
|
219
329
|
|
220
|
-
### Iterable Steps
|
221
|
-
|
222
|
-
Sometimes our workflows have steps that need to iterate over a collection and perform an action for each item in the collection before moving on to the next step in the workflow. In these cases, we can use the `for_each` option when defining our step to specific the collection, and `acidic_job` will pass each item into your step method for processing, keeping the same transactional guarantees as for any step. This means that if your step encounters an error in processing any item in the collection, when your job is retried, the job will jump right back to that step and right back to that item in the collection to try again.
|
223
|
-
|
224
|
-
```ruby
|
225
|
-
class ExampleJob < ActiveJob::Base
|
226
|
-
include AcidicJob
|
227
|
-
|
228
|
-
def perform(record:)
|
229
|
-
with_acidity providing: { collection: [1, 2, 3, 4, 5] } do
|
230
|
-
step :process_item, for_each: :collection
|
231
|
-
step :next_step
|
232
|
-
end
|
233
|
-
end
|
234
|
-
|
235
|
-
def process_item(item)
|
236
|
-
# do whatever work needs to be done with this individual item
|
237
|
-
end
|
238
|
-
end
|
239
|
-
```
|
240
|
-
|
241
|
-
**Note:** The same restrictions apply here as for any persisted attribute — you can only use objects that can be serialized by ActiveRecord.
|
242
330
|
|
243
331
|
### Sidekiq Callbacks
|
244
332
|
|
@@ -246,78 +334,6 @@ In order to ensure that `AcidicJob::Staged` records are only destroyed once the
|
|
246
334
|
|
247
335
|
This allows `acidic_job` to use an `after_perform` callback to delete the `AcidicJob::Staged` record, whether you are using the gem with ActiveJob or pure Sidekiq Workers. Of course, this means that you can add your own callbacks to any jobs or workers that include the `AcidicJob` module as well.
|
248
336
|
|
249
|
-
### Sidekiq Batches
|
250
|
-
|
251
|
-
One final feature for those of you using Sidekiq Pro: an integrated DSL for Sidekiq Batches. By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous workers to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different workers enqueued on 3 different queues, each running in parallel. Once all 3 workers succeed, `acidic_job` will move on to step 2. That's right, by leveraging the power of Sidekiq Batches, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations, and in my mind is the number one selling point for Sidekiq Pro.
|
252
|
-
|
253
|
-
In my opinion, any commercial software using Sidekiq should get Sidekiq Pro; it is _absolutely_ worth the money. If, however, you are using `acidic_job` in a non-commercial application, you could use the open-source dropin replacement for this functionality: https://github.com/breamware/sidekiq-batch
|
254
|
-
|
255
|
-
```ruby
|
256
|
-
class RideCreateJob < ActiveJob::Base
|
257
|
-
include AcidicJob
|
258
|
-
|
259
|
-
def perform(user_id, ride_params)
|
260
|
-
@user = User.find(user_id)
|
261
|
-
@params = ride_params
|
262
|
-
|
263
|
-
with_acidity providing: { ride: nil } do
|
264
|
-
step :create_ride_and_audit_record, awaits: [SomeJob]
|
265
|
-
step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
|
266
|
-
step :send_receipt
|
267
|
-
end
|
268
|
-
end
|
269
|
-
end
|
270
|
-
```
|
271
|
-
|
272
|
-
If you need to await a job that takes arguments, you can prepare that job along with its arguments using the `with` class method that `acidic_job` will add to your jobs:
|
273
|
-
|
274
|
-
```ruby
|
275
|
-
class RideCreateJob < ActiveJob::Base
|
276
|
-
include AcidicJob
|
277
|
-
|
278
|
-
def perform(user_id, ride_params)
|
279
|
-
@user = User.find(user_id)
|
280
|
-
@params = ride_params
|
281
|
-
|
282
|
-
with_acidity providing: { ride: nil } do
|
283
|
-
step :create_ride_and_audit_record, awaits: awaits: [SomeJob.with('argument_1', keyword: 'value')]
|
284
|
-
step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
|
285
|
-
step :send_receipt
|
286
|
-
end
|
287
|
-
end
|
288
|
-
end
|
289
|
-
```
|
290
|
-
|
291
|
-
You can also await a batch of jobs by simply passing multiple jobs to the `awaits` array (e.g. `awaits: [SomeJob, AnotherJob.with('argument_1', keyword: 'value')]`). Your top level workflow job will only continue to the next step once all of the jobs in your `awaits` array have successfully finished.
|
292
|
-
|
293
|
-
In some cases, you may need to _dynamically_ determine the collection of jobs that the step should wait for; in these cases, you can pass the name of a method to the `awaits` option:
|
294
|
-
|
295
|
-
```ruby
|
296
|
-
class RideCreateJob < ActiveJob::Base
|
297
|
-
include AcidicJob
|
298
|
-
set_callback :finish, :after, :delete_run_record
|
299
|
-
|
300
|
-
def perform(user_id, ride_params)
|
301
|
-
@user = User.find(user_id)
|
302
|
-
@params = ride_params
|
303
|
-
|
304
|
-
with_acidity providing: { ride: nil } do
|
305
|
-
step :create_ride_and_audit_record, awaits: :dynamic_awaits
|
306
|
-
step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
|
307
|
-
step :send_receipt
|
308
|
-
end
|
309
|
-
end
|
310
|
-
|
311
|
-
def dynamic_awaits
|
312
|
-
if @params["key"].present?
|
313
|
-
[SomeJob.with('argument_1', keyword: 'value')]
|
314
|
-
else
|
315
|
-
[AnotherJob]
|
316
|
-
end
|
317
|
-
end
|
318
|
-
end
|
319
|
-
```
|
320
|
-
|
321
337
|
|
322
338
|
### Run Finished Callbacks
|
323
339
|
|
@@ -349,6 +365,7 @@ class RideCreateJob < ActiveJob::Base
|
|
349
365
|
end
|
350
366
|
```
|
351
367
|
|
368
|
+
|
352
369
|
## Testing
|
353
370
|
|
354
371
|
When testing acidic jobs, you are likely to run into `ActiveRecord::TransactionIsolationError`s:
|
data/acidic_job.gemspec
CHANGED
@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
|
|
35
35
|
spec.add_development_dependency "minitest"
|
36
36
|
spec.add_development_dependency "net-smtp"
|
37
37
|
spec.add_development_dependency "noticed"
|
38
|
+
spec.add_development_dependency "psych", "> 4.0"
|
38
39
|
spec.add_development_dependency "railties"
|
39
40
|
spec.add_development_dependency "rake"
|
40
41
|
spec.add_development_dependency "rubocop"
|
File without changes
|
data/lib/acidic_job/awaiting.rb
CHANGED
@@ -47,7 +47,7 @@ module AcidicJob
|
|
47
47
|
arguments = args || @args
|
48
48
|
arguments += [kwargs] unless kwargs.empty?
|
49
49
|
normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(arguments))
|
50
|
-
item = { "class" => self.class, "args" => normalized_args, "jid" => jid }
|
50
|
+
item = { "class" => self.class.name, "args" => normalized_args, "jid" => jid }
|
51
51
|
sidekiq_options = sidekiq_options_hash || {}
|
52
52
|
|
53
53
|
sidekiq_options.merge(item)
|
data/lib/acidic_job/run.rb
CHANGED
@@ -3,6 +3,7 @@
|
|
3
3
|
require "active_record"
|
4
4
|
require "global_id"
|
5
5
|
require "active_support/core_ext/object/with_options"
|
6
|
+
require_relative "./serializer"
|
6
7
|
|
7
8
|
module AcidicJob
|
8
9
|
class Run < ActiveRecord::Base
|
@@ -17,11 +18,11 @@ module AcidicJob
|
|
17
18
|
|
18
19
|
after_create_commit :enqueue_staged_job, if: :staged?
|
19
20
|
|
20
|
-
serialize :
|
21
|
-
serialize :
|
22
|
-
serialize :workflow
|
23
|
-
serialize :returning_to
|
24
|
-
store :attr_accessors
|
21
|
+
serialize :serialized_job, JSON
|
22
|
+
serialize :error_object, Serializer
|
23
|
+
serialize :workflow, Serializer
|
24
|
+
serialize :returning_to, Serializer
|
25
|
+
store :attr_accessors, coder: Serializer
|
25
26
|
|
26
27
|
validates :staged, inclusion: { in: [true, false] } # uses database default
|
27
28
|
validates :serialized_job, presence: true
|
@@ -0,0 +1,163 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_job/serializers"
|
4
|
+
require "active_job/arguments"
|
5
|
+
require "json"
|
6
|
+
|
7
|
+
class WorkerSerializer < ActiveJob::Serializers::ObjectSerializer
|
8
|
+
def serialize(worker)
|
9
|
+
super(
|
10
|
+
"class" => worker.class.name,
|
11
|
+
"args" => worker.instance_variable_get(:@__acidic_job_args),
|
12
|
+
"kwargs" => worker.instance_variable_get(:@__acidic_job_kwargs)
|
13
|
+
)
|
14
|
+
end
|
15
|
+
|
16
|
+
def deserialize(hash)
|
17
|
+
worker_class = hash["class"].constantize
|
18
|
+
worker_class.new(*hash["args"], **hash["kwargs"])
|
19
|
+
end
|
20
|
+
|
21
|
+
def serialize?(argument)
|
22
|
+
defined?(::Sidekiq) && argument.class.include?(::Sidekiq::Worker)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
class JobSerializer < ActiveJob::Serializers::ObjectSerializer
|
27
|
+
def serialize(job)
|
28
|
+
super(job.serialize)
|
29
|
+
end
|
30
|
+
|
31
|
+
def deserialize(hash)
|
32
|
+
job = ActiveJob::Base.deserialize(hash)
|
33
|
+
job.send(:deserialize_arguments_if_needed)
|
34
|
+
if job.arguments.last.is_a?(Hash)
|
35
|
+
*args, kwargs = job.arguments
|
36
|
+
else
|
37
|
+
args = job.arguments
|
38
|
+
kwargs = {}
|
39
|
+
end
|
40
|
+
job.instance_variable_set(:@__acidic_job_args, args)
|
41
|
+
job.instance_variable_set(:@__acidic_job_kwargs, kwargs)
|
42
|
+
|
43
|
+
job
|
44
|
+
end
|
45
|
+
|
46
|
+
def serialize?(argument)
|
47
|
+
defined?(::ActiveJob::Base) && argument.class < ::ActiveJob::Base
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
class ExceptionSerializer < ActiveJob::Serializers::ObjectSerializer
|
52
|
+
def serialize(exception)
|
53
|
+
hash = {
|
54
|
+
"class" => exception.class.name,
|
55
|
+
"message" => exception.message,
|
56
|
+
"cause" => exception.cause,
|
57
|
+
"backtrace" => {}
|
58
|
+
}
|
59
|
+
|
60
|
+
exception.backtrace.map do |trace|
|
61
|
+
path, _, location = trace.rpartition("/")
|
62
|
+
|
63
|
+
next if hash["backtrace"].key?(path)
|
64
|
+
|
65
|
+
hash["backtrace"][path] = location
|
66
|
+
end
|
67
|
+
|
68
|
+
super(hash)
|
69
|
+
end
|
70
|
+
|
71
|
+
def deserialize(hash)
|
72
|
+
exception_class = hash["class"].constantize
|
73
|
+
exception = exception_class.new(hash["message"])
|
74
|
+
exception.set_backtrace(hash["backtrace"].map do |path, location|
|
75
|
+
[path, location].join("/")
|
76
|
+
end)
|
77
|
+
exception
|
78
|
+
end
|
79
|
+
|
80
|
+
def serialize?(argument)
|
81
|
+
defined?(Exception) && argument.is_a?(Exception)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
class FinishedPointSerializer < ActiveJob::Serializers::ObjectSerializer
|
86
|
+
def serialize(finished_point)
|
87
|
+
super(
|
88
|
+
"class" => finished_point.class.name
|
89
|
+
)
|
90
|
+
end
|
91
|
+
|
92
|
+
def deserialize(hash)
|
93
|
+
finished_point_class = hash["class"].constantize
|
94
|
+
finished_point_class.new
|
95
|
+
end
|
96
|
+
|
97
|
+
def serialize?(argument)
|
98
|
+
defined?(::AcidicJob::FinishedPoint) && argument.is_a?(::AcidicJob::FinishedPoint)
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
class RecoveryPointSerializer < ActiveJob::Serializers::ObjectSerializer
|
103
|
+
def serialize(recovery_point)
|
104
|
+
super(
|
105
|
+
"class" => recovery_point.class.name,
|
106
|
+
"name" => recovery_point.name
|
107
|
+
)
|
108
|
+
end
|
109
|
+
|
110
|
+
def deserialize(hash)
|
111
|
+
recovery_point_class = hash["class"].constantize
|
112
|
+
recovery_point_class.new(hash["name"])
|
113
|
+
end
|
114
|
+
|
115
|
+
def serialize?(argument)
|
116
|
+
defined?(::AcidicJob::RecoveryPoint) && argument.is_a?(::AcidicJob::RecoveryPoint)
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
ActiveJob::Serializers.add_serializers(
|
121
|
+
WorkerSerializer,
|
122
|
+
JobSerializer,
|
123
|
+
ExceptionSerializer,
|
124
|
+
FinishedPointSerializer,
|
125
|
+
RecoveryPointSerializer
|
126
|
+
)
|
127
|
+
|
128
|
+
# ...
|
129
|
+
module AcidicJob
|
130
|
+
module Arguments
|
131
|
+
include ActiveJob::Arguments
|
132
|
+
extend self # rubocop:disable Style/ModuleFunction
|
133
|
+
|
134
|
+
# `ActiveJob` will throw an error if it tries to deserialize a GlobalID record.
|
135
|
+
# However, this isn't the behavior that we want for our custom `ActiveRecord` serializer.
|
136
|
+
# Since `ActiveRecord` does _not_ reset instance record state to its pre-transactional state
|
137
|
+
# on a transaction ROLLBACK, we can have GlobalID entries in a serialized column that point to
|
138
|
+
# non-persisted records. This is ok. We should simply return `nil` for that portion of the
|
139
|
+
# serialized field.
|
140
|
+
def deserialize_global_id(hash)
|
141
|
+
GlobalID::Locator.locate hash[GLOBALID_KEY]
|
142
|
+
rescue ActiveRecord::RecordNotFound
|
143
|
+
nil
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
class Serializer
|
148
|
+
# Used for `serialize` method in ActiveRecord
|
149
|
+
class << self
|
150
|
+
def load(json)
|
151
|
+
return if json.nil? || json.empty?
|
152
|
+
|
153
|
+
data = JSON.parse(json)
|
154
|
+
Arguments.deserialize(data).first
|
155
|
+
end
|
156
|
+
|
157
|
+
def dump(obj)
|
158
|
+
data = Arguments.serialize [obj]
|
159
|
+
data.to_json
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
163
|
+
end
|
data/lib/acidic_job/version.rb
CHANGED
data/lib/acidic_job.rb
CHANGED
@@ -160,7 +160,7 @@ module AcidicJob
|
|
160
160
|
break
|
161
161
|
elsif current_step.nil?
|
162
162
|
raise UnknownRecoveryPoint, "Defined workflow does not reference this step: #{recovery_point}"
|
163
|
-
elsif !(jobs = current_step.fetch("awaits", []) || []).empty?
|
163
|
+
elsif !Array(jobs = current_step.fetch("awaits", []) || []).compact.empty?
|
164
164
|
step = Step.new(current_step, run, self)
|
165
165
|
# Only execute the current step, without yet progressing the recovery_point to the next step.
|
166
166
|
# This ensures that any failures in parallel jobs will have this step retried in the main workflow
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: acidic_job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.0.
|
4
|
+
version: 1.0.0.pre28
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fractaledmind
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-
|
11
|
+
date: 2022-08-03 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -122,6 +122,20 @@ dependencies:
|
|
122
122
|
- - ">="
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: psych
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - ">"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: '4.0'
|
132
|
+
type: :development
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - ">"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: '4.0'
|
125
139
|
- !ruby/object:Gem::Dependency
|
126
140
|
name: railties
|
127
141
|
requirement: !ruby/object:Gem::Requirement
|
@@ -270,6 +284,7 @@ files:
|
|
270
284
|
- bin/console
|
271
285
|
- bin/setup
|
272
286
|
- blog_post.md
|
287
|
+
- combustion/log/test.log
|
273
288
|
- gemfiles/rails_6.1.gemfile
|
274
289
|
- gemfiles/rails_7.0.gemfile
|
275
290
|
- lib/acidic_job.rb
|
@@ -285,6 +300,7 @@ files:
|
|
285
300
|
- lib/acidic_job/recovery_point.rb
|
286
301
|
- lib/acidic_job/rspec_configuration.rb
|
287
302
|
- lib/acidic_job/run.rb
|
303
|
+
- lib/acidic_job/serializer.rb
|
288
304
|
- lib/acidic_job/staging.rb
|
289
305
|
- lib/acidic_job/step.rb
|
290
306
|
- lib/acidic_job/test_case.rb
|