acidic_job 0.7.7 → 1.0.0.pre3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +21 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +99 -3
- data/README.md +47 -14
- data/UPGRADE_GUIDE.md +71 -0
- data/acidic_job.gemspec +2 -2
- data/bin/console +3 -2
- data/lib/acidic_job/awaiting.rb +68 -0
- data/lib/acidic_job/errors.rb +2 -0
- data/lib/acidic_job/extensions/action_mailer.rb +27 -0
- data/lib/acidic_job/extensions/active_job.rb +39 -0
- data/lib/acidic_job/extensions/noticed.rb +52 -0
- data/lib/acidic_job/extensions/sidekiq.rb +101 -0
- data/lib/acidic_job/{response.rb → finished_point.rb} +4 -4
- data/lib/acidic_job/idempotency_key.rb +24 -0
- data/lib/acidic_job/perform_wrapper.rb +34 -20
- data/lib/acidic_job/recovery_point.rb +3 -3
- data/lib/acidic_job/run.rb +77 -0
- data/lib/acidic_job/staging.rb +30 -0
- data/lib/acidic_job/step.rb +83 -0
- data/lib/acidic_job/upgrade_service.rb +115 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +121 -205
- data/lib/generators/acidic_job/drop_tables_generator.rb +31 -0
- data/lib/generators/acidic_job_generator.rb +5 -24
- data/lib/generators/templates/create_acidic_job_runs_migration.rb.erb +19 -0
- data/lib/generators/templates/{create_acidic_job_keys_migration.rb.erb → drop_acidic_job_keys_migration.rb.erb} +10 -3
- metadata +23 -17
- data/lib/acidic_job/deliver_transactionally_extension.rb +0 -26
- data/lib/acidic_job/key.rb +0 -33
- data/lib/acidic_job/no_op.rb +0 -11
- data/lib/acidic_job/perform_transactionally_extension.rb +0 -33
- data/lib/acidic_job/sidekiq_callbacks.rb +0 -45
- data/lib/acidic_job/staged.rb +0 -50
- data/lib/generators/templates/create_staged_acidic_jobs_migration.rb.erb +0 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7c1aec259fa05e5cd62425643534a6944b44df91e1077dfb2eb98fa0717c3e1
|
4
|
+
data.tar.gz: 1e9759e91c7a10ea89f39626add3c60b5c3856d54a380958b9f4e6455c6cc937
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 53d61ee2cee38e5e4a136eb37386c2bb912933ba35f839949ac411c486dd14091478d5d23ae3bb2d4ebd3f4c2d9f051e77a64ef1113f8fec2e533f7209662cdd
|
7
|
+
data.tar.gz: b8e4dcb13d2f4551d1cc37271003eff592bc03791c77edd3224d306d211c240b850ef258d184155511c29983af349ef9039b4b2dab99d298cc925a36217d7f11
|
data/.gitignore
CHANGED
data/.rubocop.yml
CHANGED
@@ -15,3 +15,24 @@ Layout/LineLength:
|
|
15
15
|
|
16
16
|
Style/Documentation:
|
17
17
|
Enabled: false
|
18
|
+
|
19
|
+
Metrics/ModuleLength:
|
20
|
+
Enabled: false
|
21
|
+
|
22
|
+
Metrics/AbcSize:
|
23
|
+
Enabled: false
|
24
|
+
|
25
|
+
Metrics/MethodLength:
|
26
|
+
Enabled: false
|
27
|
+
|
28
|
+
Metrics/BlockLength:
|
29
|
+
Enabled: false
|
30
|
+
|
31
|
+
Metrics/CyclomaticComplexity:
|
32
|
+
Enabled: false
|
33
|
+
|
34
|
+
Metrics/PerceivedComplexity:
|
35
|
+
Enabled: false
|
36
|
+
|
37
|
+
Metrics/ClassLength:
|
38
|
+
Enabled: false
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,13 +1,32 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
acidic_job (0.
|
5
|
-
activerecord (>=
|
4
|
+
acidic_job (1.0.0.pre3)
|
5
|
+
activerecord (>= 6.1.0)
|
6
6
|
activesupport
|
7
7
|
|
8
8
|
GEM
|
9
9
|
remote: https://rubygems.org/
|
10
10
|
specs:
|
11
|
+
actioncable (6.1.3.2)
|
12
|
+
actionpack (= 6.1.3.2)
|
13
|
+
activesupport (= 6.1.3.2)
|
14
|
+
nio4r (~> 2.0)
|
15
|
+
websocket-driver (>= 0.6.1)
|
16
|
+
actionmailbox (6.1.3.2)
|
17
|
+
actionpack (= 6.1.3.2)
|
18
|
+
activejob (= 6.1.3.2)
|
19
|
+
activerecord (= 6.1.3.2)
|
20
|
+
activestorage (= 6.1.3.2)
|
21
|
+
activesupport (= 6.1.3.2)
|
22
|
+
mail (>= 2.7.1)
|
23
|
+
actionmailer (6.1.3.2)
|
24
|
+
actionpack (= 6.1.3.2)
|
25
|
+
actionview (= 6.1.3.2)
|
26
|
+
activejob (= 6.1.3.2)
|
27
|
+
activesupport (= 6.1.3.2)
|
28
|
+
mail (~> 2.5, >= 2.5.4)
|
29
|
+
rails-dom-testing (~> 2.0)
|
11
30
|
actionpack (6.1.3.2)
|
12
31
|
actionview (= 6.1.3.2)
|
13
32
|
activesupport (= 6.1.3.2)
|
@@ -15,6 +34,12 @@ GEM
|
|
15
34
|
rack-test (>= 0.6.3)
|
16
35
|
rails-dom-testing (~> 2.0)
|
17
36
|
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
37
|
+
actiontext (6.1.3.2)
|
38
|
+
actionpack (= 6.1.3.2)
|
39
|
+
activerecord (= 6.1.3.2)
|
40
|
+
activestorage (= 6.1.3.2)
|
41
|
+
activesupport (= 6.1.3.2)
|
42
|
+
nokogiri (>= 1.8.5)
|
18
43
|
actionview (6.1.3.2)
|
19
44
|
activesupport (= 6.1.3.2)
|
20
45
|
builder (~> 3.1)
|
@@ -29,15 +54,28 @@ GEM
|
|
29
54
|
activerecord (6.1.3.2)
|
30
55
|
activemodel (= 6.1.3.2)
|
31
56
|
activesupport (= 6.1.3.2)
|
57
|
+
activestorage (6.1.3.2)
|
58
|
+
actionpack (= 6.1.3.2)
|
59
|
+
activejob (= 6.1.3.2)
|
60
|
+
activerecord (= 6.1.3.2)
|
61
|
+
activesupport (= 6.1.3.2)
|
62
|
+
marcel (~> 1.0.0)
|
63
|
+
mini_mime (~> 1.0.2)
|
32
64
|
activesupport (6.1.3.2)
|
33
65
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
34
66
|
i18n (>= 1.6, < 2)
|
35
67
|
minitest (>= 5.1)
|
36
68
|
tzinfo (~> 2.0)
|
37
69
|
zeitwerk (~> 2.3)
|
70
|
+
addressable (2.8.0)
|
71
|
+
public_suffix (>= 2.0.2, < 5.0)
|
38
72
|
ast (2.4.2)
|
39
73
|
builder (3.2.4)
|
40
74
|
coderay (1.1.3)
|
75
|
+
combustion (1.3.5)
|
76
|
+
activesupport (>= 3.0.0)
|
77
|
+
railties (>= 3.0.0)
|
78
|
+
thor (>= 0.14.6)
|
41
79
|
concurrent-ruby (1.1.9)
|
42
80
|
connection_pool (2.2.5)
|
43
81
|
crass (1.0.6)
|
@@ -48,30 +86,71 @@ GEM
|
|
48
86
|
database_cleaner-core (~> 2.0.0)
|
49
87
|
database_cleaner-core (2.0.1)
|
50
88
|
docile (1.4.0)
|
89
|
+
domain_name (0.5.20190701)
|
90
|
+
unf (>= 0.0.5, < 1.0.0)
|
51
91
|
erubi (1.10.0)
|
92
|
+
ffi (1.15.5)
|
93
|
+
ffi-compiler (1.0.1)
|
94
|
+
ffi (>= 1.0.0)
|
95
|
+
rake
|
52
96
|
globalid (0.4.2)
|
53
97
|
activesupport (>= 4.2.0)
|
98
|
+
http (5.0.4)
|
99
|
+
addressable (~> 2.8)
|
100
|
+
http-cookie (~> 1.0)
|
101
|
+
http-form_data (~> 2.2)
|
102
|
+
llhttp-ffi (~> 0.4.0)
|
103
|
+
http-cookie (1.0.4)
|
104
|
+
domain_name (~> 0.5)
|
105
|
+
http-form_data (2.3.0)
|
54
106
|
i18n (1.8.10)
|
55
107
|
concurrent-ruby (~> 1.0)
|
108
|
+
llhttp-ffi (0.4.0)
|
109
|
+
ffi-compiler (~> 1.0)
|
110
|
+
rake (~> 13.0)
|
56
111
|
loofah (2.12.0)
|
57
112
|
crass (~> 1.0.2)
|
58
113
|
nokogiri (>= 1.5.9)
|
114
|
+
mail (2.7.1)
|
115
|
+
mini_mime (>= 0.1.1)
|
116
|
+
marcel (1.0.2)
|
59
117
|
method_source (1.0.0)
|
118
|
+
mini_mime (1.0.3)
|
60
119
|
mini_portile2 (2.6.1)
|
61
120
|
minitest (5.14.4)
|
121
|
+
nio4r (2.5.8)
|
62
122
|
nokogiri (1.12.3)
|
63
123
|
mini_portile2 (~> 2.6.1)
|
64
124
|
racc (~> 1.4)
|
125
|
+
noticed (1.5.7)
|
126
|
+
http (>= 4.0.0)
|
127
|
+
rails (>= 5.2.0)
|
65
128
|
parallel (1.20.1)
|
66
129
|
parser (3.0.1.1)
|
67
130
|
ast (~> 2.4.1)
|
68
131
|
pry (0.14.1)
|
69
132
|
coderay (~> 1.1)
|
70
133
|
method_source (~> 1.0)
|
134
|
+
public_suffix (4.0.6)
|
71
135
|
racc (1.5.2)
|
72
136
|
rack (2.2.3)
|
73
137
|
rack-test (1.1.0)
|
74
138
|
rack (>= 1.0, < 3)
|
139
|
+
rails (6.1.3.2)
|
140
|
+
actioncable (= 6.1.3.2)
|
141
|
+
actionmailbox (= 6.1.3.2)
|
142
|
+
actionmailer (= 6.1.3.2)
|
143
|
+
actionpack (= 6.1.3.2)
|
144
|
+
actiontext (= 6.1.3.2)
|
145
|
+
actionview (= 6.1.3.2)
|
146
|
+
activejob (= 6.1.3.2)
|
147
|
+
activemodel (= 6.1.3.2)
|
148
|
+
activerecord (= 6.1.3.2)
|
149
|
+
activestorage (= 6.1.3.2)
|
150
|
+
activesupport (= 6.1.3.2)
|
151
|
+
bundler (>= 1.15.0)
|
152
|
+
railties (= 6.1.3.2)
|
153
|
+
sprockets-rails (>= 2.0.0)
|
75
154
|
rails-dom-testing (2.0.3)
|
76
155
|
activesupport (>= 4.2.0)
|
77
156
|
nokogiri (>= 1.6)
|
@@ -114,11 +193,25 @@ GEM
|
|
114
193
|
simplecov_json_formatter (~> 0.1)
|
115
194
|
simplecov-html (0.12.3)
|
116
195
|
simplecov_json_formatter (0.1.3)
|
196
|
+
sprockets (4.0.3)
|
197
|
+
concurrent-ruby (~> 1.0)
|
198
|
+
rack (> 1, < 3)
|
199
|
+
sprockets-rails (3.4.2)
|
200
|
+
actionpack (>= 5.2)
|
201
|
+
activesupport (>= 5.2)
|
202
|
+
sprockets (>= 3.0.0)
|
117
203
|
sqlite3 (1.4.2)
|
118
204
|
thor (1.1.0)
|
119
205
|
tzinfo (2.0.4)
|
120
206
|
concurrent-ruby (~> 1.0)
|
207
|
+
unf (0.1.4)
|
208
|
+
unf_ext
|
209
|
+
unf_ext (0.0.8)
|
121
210
|
unicode-display_width (2.0.0)
|
211
|
+
warning (1.2.1)
|
212
|
+
websocket-driver (0.7.5)
|
213
|
+
websocket-extensions (>= 0.1.0)
|
214
|
+
websocket-extensions (0.1.5)
|
122
215
|
zeitwerk (2.4.2)
|
123
216
|
|
124
217
|
PLATFORMS
|
@@ -129,10 +222,12 @@ DEPENDENCIES
|
|
129
222
|
acidic_job!
|
130
223
|
activejob (~> 6.1.3.2)
|
131
224
|
activerecord (~> 6.1.3.2)
|
225
|
+
combustion
|
132
226
|
database_cleaner
|
133
227
|
minitest (~> 5.0)
|
228
|
+
noticed
|
134
229
|
pry
|
135
|
-
railties (>=
|
230
|
+
railties (>= 6.1.0)
|
136
231
|
rake (~> 13.0)
|
137
232
|
rubocop (~> 1.7)
|
138
233
|
rubocop-minitest
|
@@ -140,6 +235,7 @@ DEPENDENCIES
|
|
140
235
|
sidekiq
|
141
236
|
simplecov
|
142
237
|
sqlite3
|
238
|
+
warning
|
143
239
|
|
144
240
|
BUNDLED WITH
|
145
241
|
2.2.31
|
data/README.md
CHANGED
@@ -37,7 +37,7 @@ Or simply execute to install the gem yourself:
|
|
37
37
|
|
38
38
|
$ bundle add acidic_job
|
39
39
|
|
40
|
-
Then, use the following command to copy over the `AcidicJob::
|
40
|
+
Then, use the following command to copy over the `AcidicJob::Run` migration file.
|
41
41
|
|
42
42
|
```
|
43
43
|
rails generate acidic_job
|
@@ -45,16 +45,28 @@ rails generate acidic_job
|
|
45
45
|
|
46
46
|
## Usage
|
47
47
|
|
48
|
-
`AcidicJob` is a concern that you `include` into your
|
48
|
+
`AcidicJob` is a concern that you `include` into your base `ApplicationJob`.
|
49
49
|
|
50
50
|
```ruby
|
51
|
-
class
|
51
|
+
class ApplicationJob < ActiveJob::Base
|
52
52
|
include AcidicJob
|
53
53
|
end
|
54
54
|
```
|
55
55
|
|
56
|
+
This is useful because the module needs to be mixed into any and all jobs that you want to either make acidic or enqueue acidicly.
|
57
|
+
|
56
58
|
It provides a suite of functionality that empowers you to create complex, robust, and _acidic_ jobs.
|
57
59
|
|
60
|
+
### TL;DR
|
61
|
+
|
62
|
+
#### Key Features
|
63
|
+
|
64
|
+
* Transactional Steps — break your job into a series of steps, each of which will be run within an acidic database transaction, allowing retries to jump back to the last "recovery point".
|
65
|
+
* Persisted Attributes — when retrying jobs at later steps, we need to ensure that data created in previous steps is still available to later steps on retry.
|
66
|
+
* Transactionally Staged Jobs — enqueue additional jobs within the acidic transaction safely
|
67
|
+
* Sidekiq Callbacks — bring ActiveJob-like callbacks into your pure Sidekiq Workers
|
68
|
+
* Sidekiq Batches — leverage the power of Sidekiq Pro's `batch` functionality without the hassle
|
69
|
+
|
58
70
|
### Transactional Steps
|
59
71
|
|
60
72
|
The first and foundational feature `acidic_job` provides is the `with_acidity` method, which takes a block of transactional step methods (defined via the `step`) method:
|
@@ -63,8 +75,10 @@ The first and foundational feature `acidic_job` provides is the `with_acidity` m
|
|
63
75
|
class RideCreateJob < ActiveJob::Base
|
64
76
|
include AcidicJob
|
65
77
|
|
66
|
-
def perform(ride_params)
|
67
|
-
|
78
|
+
def perform(user_id, ride_params)
|
79
|
+
user = User.find(user_id)
|
80
|
+
|
81
|
+
with_acidity given: { user: user, params: ride_params, ride: nil } do
|
68
82
|
step :create_ride_and_audit_record
|
69
83
|
step :create_stripe_charge
|
70
84
|
step :send_receipt
|
@@ -87,11 +101,11 @@ end
|
|
87
101
|
|
88
102
|
`with_acidity` takes only the `given:` named parameter and a block where you define the steps of this operation. `step` simply takes the name of a method available in the job. That's all!
|
89
103
|
|
90
|
-
Now, each execution of this job will find or create an `AcidicJob::
|
104
|
+
Now, each execution of this job will find or create an `AcidicJob::Run` record, which we leverage to wrap every step in a database transaction. Moreover, this database record allows `acidic_job` to ensure that if your job fails on step 3, when it retries, it will simply jump right back to trying to execute the method defined for the 3rd step, and won't even execute the first two step methods. This means your step methods only need to be idempotent on failure, not on success, since they will never be run again if they succeed.
|
91
105
|
|
92
106
|
### Persisted Attributes
|
93
107
|
|
94
|
-
Any objects passed to the `given` option on the `with_acidity` method are not just made available to each of your step methods, they are made available across retries. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::
|
108
|
+
Any objects passed to the `given` option on the `with_acidity` method are not just made available to each of your step methods, they are made available across retries. This means that you can set an attribute in step 1, access it in step 2, have step 2 fail, have the job retry, jump directly back to step 2 on retry, and have that object still accessible. This is done by serializing all objects to a field on the `AcidicJob::Run` and manually providing getters and setters that sync with the database record.
|
95
109
|
|
96
110
|
```ruby
|
97
111
|
class RideCreateJob < ActiveJob::Base
|
@@ -110,7 +124,7 @@ class RideCreateJob < ActiveJob::Base
|
|
110
124
|
end
|
111
125
|
|
112
126
|
def create_stripe_charge
|
113
|
-
Stripe::Charge.create(amount: 20_00, customer:
|
127
|
+
Stripe::Charge.create(amount: 20_00, customer: self.ride.user)
|
114
128
|
end
|
115
129
|
|
116
130
|
# ...
|
@@ -119,20 +133,22 @@ end
|
|
119
133
|
|
120
134
|
**Note:** This does mean that you are restricted to objects that can be serialized by ActiveRecord, thus no Procs, for example.
|
121
135
|
|
122
|
-
**Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the
|
136
|
+
**Note:** You will note the use of `self.ride = ...` in the code sample above. In order to call the attribute setter method that will sync with the database record, you _must_ use this style. `@ride = ...` and/or `ride = ...` will both fail to sync the value with the database record.
|
123
137
|
|
124
138
|
### Transactionally Staged Jobs
|
125
139
|
|
126
140
|
A standard problem when inside of database transactions is enqueuing other jobs. On the one hand, you could enqueue a job inside of a transaction that then rollbacks, which would leave that job to fail and retry and fail. On the other hand, you could enqueue a job that is picked up before the transaction commits, which would mean the records are not yet available to this job.
|
127
141
|
|
128
|
-
In order to mitigate against such issues without forcing you to use a database-backed job queue, `acidic_job` provides `
|
142
|
+
In order to mitigate against such issues without forcing you to use a database-backed job queue, `acidic_job` provides `perform_acidicly` and `deliver_acidicly` methods to "transactionally stage" enqueuing other jobs from within a step (whether another ActiveJob or a Sidekiq::Worker or an ActionMailer delivery). These methods will create a new `AcidicJob::Run` record, but inside of the database transaction of the `step`. Upon commit of that transaction, a model callback pushes the job to your actual job queue. Once the job has been successfully performed, the `AcidicJob::Run` record is deleted so that this table doesn't grow unbounded and unnecessarily.
|
129
143
|
|
130
144
|
```ruby
|
131
145
|
class RideCreateJob < ActiveJob::Base
|
132
146
|
include AcidicJob
|
133
147
|
|
134
|
-
def perform(ride_params)
|
135
|
-
|
148
|
+
def perform(user_id, ride_params)
|
149
|
+
user = User.find(user_id)
|
150
|
+
|
151
|
+
with_acidity given: { user: user, params: ride_params, ride: nil } do
|
136
152
|
step :create_ride_and_audit_record
|
137
153
|
step :create_stripe_charge
|
138
154
|
step :send_receipt
|
@@ -142,7 +158,7 @@ class RideCreateJob < ActiveJob::Base
|
|
142
158
|
# ...
|
143
159
|
|
144
160
|
def send_receipt
|
145
|
-
RideMailer.with(
|
161
|
+
RideMailer.with(user: @user, ride: @ride).confirm_charge.delivery_acidicly
|
146
162
|
end
|
147
163
|
end
|
148
164
|
```
|
@@ -155,10 +171,27 @@ This allows `acidic_job` to use an `after_perform` callback to delete the `Acidi
|
|
155
171
|
|
156
172
|
### Sidekiq Batches
|
157
173
|
|
158
|
-
One final feature for those of you using Sidekiq Pro: an integrated DSL for Sidekiq Batches. By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous workers to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different workers enqueued on 3 different queues, each running in parallel. Once all 3 workers succeed, `acidic_job` will move on to step 2. That's right, by leveraging the power of Sidekiq Batches, you can have workers that are
|
174
|
+
One final feature for those of you using Sidekiq Pro: an integrated DSL for Sidekiq Batches. By simply adding the `awaits` option to your step declarations, you can attach any number of additional, asynchronous workers to your step. This is profoundly powerful, as it means that you can define a workflow where step 2 is started _if and only if_ step 1 succeeds, but step 1 can have 3 different workers enqueued on 3 different queues, each running in parallel. Once all 3 workers succeed, `acidic_job` will move on to step 2. That's right, by leveraging the power of Sidekiq Batches, you can have workers that are _executed in parallel_, **on separate queues**, and _asynchronously_, but are still **blocking**—as a group—the next step in your workflow! This unlocks incredible power and flexibility for defining and structuring complex workflows and operations, and in my mind is the number one selling point for Sidekiq Pro.
|
159
175
|
|
160
176
|
In my opinion, any commercial software using Sidekiq should get Sidekiq Pro; it is _absolutely_ worth the money. If, however, you are using `acidic_job` in a non-commercial application, you could use the open-source dropin replacement for this functionality: https://github.com/breamware/sidekiq-batch
|
161
177
|
|
178
|
+
```ruby
|
179
|
+
# TODO: write code sample
|
180
|
+
class RideCreateJob < ActiveJob::Base
|
181
|
+
include AcidicJob
|
182
|
+
|
183
|
+
def perform(user_id, ride_params)
|
184
|
+
user = User.find(user_id)
|
185
|
+
|
186
|
+
with_acidity given: { user: user, params: ride_params, ride: nil } do
|
187
|
+
step :create_ride_and_audit_record, awaits: [SomeJob]
|
188
|
+
step :create_stripe_charge, args: [1, 2, 3], kwargs: { some: 'thing' }
|
189
|
+
step :send_receipt
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
```
|
194
|
+
|
162
195
|
## Development
|
163
196
|
|
164
197
|
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
data/UPGRADE_GUIDE.md
ADDED
@@ -0,0 +1,71 @@
|
|
1
|
+
# AcidicJob Upgrade Guide
|
2
|
+
|
3
|
+
1. Update version requirements in `Gemfile`
|
4
|
+
|
5
|
+
```ruby
|
6
|
+
-gem "acidic_job"
|
7
|
+
+gem "acidic_job", "~> 1.0.0.pre1"
|
8
|
+
```
|
9
|
+
|
10
|
+
result:
|
11
|
+
```
|
12
|
+
Installing acidic_job 1.0.0.pre1 (was 0.7.7)
|
13
|
+
Bundle updated!
|
14
|
+
```
|
15
|
+
|
16
|
+
2. Generate migration for new `AcidicJob::Run` model
|
17
|
+
|
18
|
+
```bash
|
19
|
+
rails generate acidic_job
|
20
|
+
```
|
21
|
+
|
22
|
+
result:
|
23
|
+
```
|
24
|
+
create db/migrate/#{yyyymmddhhmmss}_create_acidic_job_runs.rb
|
25
|
+
```
|
26
|
+
|
27
|
+
3. Delete any unneeded `AcidicJob::Key` records
|
28
|
+
|
29
|
+
Typically, records that are already finished do not need to be retained. Sometimes, however, applications key finished records around for some amount of time for debugging or metrics aggregation. Whatever your application's logic is for whether or not an `AcidicJob::Key` record is still needed, for all unneeded records, delete them.
|
30
|
+
|
31
|
+
For example, this would delete all finished `Key` records over 1 month old:
|
32
|
+
|
33
|
+
```ruby
|
34
|
+
AcidicJob::Key.where(recovery_point: AcidicJob::Key::RECOVERY_POINT_FINISHED, last_run_at: ..1.month.ago).delete_all
|
35
|
+
```
|
36
|
+
|
37
|
+
4. Migrate `AcidicJob::Key` to `AcidicJob::Run`
|
38
|
+
|
39
|
+
`AcidicJob` ships with an upgrade module that provides a script to migrate older `Key` records to the new `Run` model.
|
40
|
+
|
41
|
+
```ruby
|
42
|
+
AcidicJob::UpgradeService.execute
|
43
|
+
```
|
44
|
+
|
45
|
+
This script will prepare an `insert_all` command for `Run` records by mapping the older `Key` data to the new `Run` schema. It also creates the new `Run` records with the same `id` as their `Key` counterparts, and then deletes all `Key` records successfully mapped over. Any `Key` records that were failed to be mapped over will be reported, along with the exception, in the `errored_keys` portion of the resulting hash.
|
46
|
+
|
47
|
+
result:
|
48
|
+
```
|
49
|
+
{
|
50
|
+
run_records: <Integer>,
|
51
|
+
key_records: <Integer>,
|
52
|
+
errored_keys: <Array>
|
53
|
+
}
|
54
|
+
```
|
55
|
+
|
56
|
+
5. Triage remaining `AcidicJob::Key` records
|
57
|
+
|
58
|
+
If there were any `AcidicJob::Key` records that failed to be mapped to the new `Run` model, you will need to manually triage whatever the exception was. In all likelihood, the exception would be relating to the translation of the `Key#job_args` field to the `Run#serialized_job` field, as all other fields have a fairly straight-forward mapping. If you can't resolve the issue, please open an Issue in GitHub.
|
59
|
+
|
60
|
+
6. Ensure all `AcidicJob::Staged` records are processed
|
61
|
+
|
62
|
+
`AcidicJob` still ships with an upgrade module that provides the older `Key` and `Staged` records, so this functionality will still be present to handle any existing records in your database when you deploy the updated version.
|
63
|
+
|
64
|
+
7. Remove the old tables
|
65
|
+
|
66
|
+
Once you have successfully migrated everything over and the new system has been running smoothly for some time, you should drop the old `acidic_job_keys` and `staged_acidic_jobs` tables. We provide a migration generator just for this purpose:
|
67
|
+
|
68
|
+
```bash
|
69
|
+
rails generate acidic_job:drop_tables
|
70
|
+
rails db:migrate
|
71
|
+
```
|
data/acidic_job.gemspec
CHANGED
@@ -27,9 +27,9 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) }
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
30
|
-
spec.add_dependency "activerecord", ">=
|
30
|
+
spec.add_dependency "activerecord", ">= 6.1.0"
|
31
31
|
spec.add_dependency "activesupport"
|
32
|
-
spec.add_development_dependency "railties", ">=
|
32
|
+
spec.add_development_dependency "railties", ">= 6.1.0"
|
33
33
|
|
34
34
|
# For more information and examples about making a new gem, checkout our
|
35
35
|
# guide at: https://bundler.io/guides/creating_gem.html
|
data/bin/console
CHANGED
@@ -3,12 +3,13 @@
|
|
3
3
|
|
4
4
|
require "bundler/setup"
|
5
5
|
require "acidic_job"
|
6
|
-
require_relative "../test/support/setup"
|
7
|
-
require_relative "../test/support/ride_create_job"
|
8
6
|
|
9
7
|
# You can add fixtures and/or initialization code here to make experimenting
|
10
8
|
# with your gem easier. You can also use a different console, if you like.
|
11
9
|
|
10
|
+
require_relative "../test/support/setup"
|
11
|
+
require_relative "../test/support/ride_create_job"
|
12
|
+
|
12
13
|
# (If you use this, don't forget to add pry to your Gemfile!)
|
13
14
|
# require "pry"
|
14
15
|
# Pry.start
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Awaiting
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
class_methods do
|
10
|
+
# TODO: Allow the `perform` method to be used to kick off Sidekiq Batch powered workflows
|
11
|
+
def initiate(*args)
|
12
|
+
raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
|
13
|
+
|
14
|
+
top_level_workflow = Sidekiq::Batch.new
|
15
|
+
top_level_workflow.on(:success, self, *args)
|
16
|
+
top_level_workflow.jobs do
|
17
|
+
perform_async
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def enqueue_step_parallel_jobs(jobs, run, step_result)
|
23
|
+
# `batch` is available from Sidekiq::Pro
|
24
|
+
raise SidekiqBatchRequired unless defined?(Sidekiq::Batch)
|
25
|
+
|
26
|
+
batch.jobs do
|
27
|
+
step_batch = Sidekiq::Batch.new
|
28
|
+
# step_batch.description = "AcidicJob::Workflow Step: #{step}"
|
29
|
+
step_batch.on(
|
30
|
+
:success,
|
31
|
+
"#{self.class.name}#step_done",
|
32
|
+
# NOTE: options are marshalled through JSON so use only basic types.
|
33
|
+
{ "run_id" => run.id,
|
34
|
+
"step_result_yaml" => step_result.to_yaml.strip }
|
35
|
+
)
|
36
|
+
# NOTE: The jobs method is atomic.
|
37
|
+
# All jobs created in the block are actually pushed atomically at the end of the block.
|
38
|
+
# If an error is raised, none of the jobs will go to Redis.
|
39
|
+
step_batch.jobs do
|
40
|
+
jobs.each do |worker_name|
|
41
|
+
# TODO: handle Symbols as well
|
42
|
+
worker = worker_name.is_a?(String) ? worker_name.constantize : worker_name
|
43
|
+
if worker.instance_method(:perform).arity.zero?
|
44
|
+
worker.perform_async
|
45
|
+
elsif worker.instance_method(:perform).arity == 1
|
46
|
+
worker.perform_async(run.id)
|
47
|
+
else
|
48
|
+
raise TooManyParametersForParallelJob
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def step_done(_status, options)
|
56
|
+
run = Run.find(options["run_id"])
|
57
|
+
current_step = run.workflow[run.recovery_point.to_s]
|
58
|
+
# re-hydrate the `step_result` object
|
59
|
+
step_result = YAML.safe_load(options["step_result_yaml"], permitted_classes: [RecoveryPoint, FinishedPoint])
|
60
|
+
step = Step.new(current_step, run, self, step_result)
|
61
|
+
|
62
|
+
# TODO: WRITE REGRESSION TESTS FOR PARALLEL JOB FAILING AND RETRYING THE ORIGINAL STEP
|
63
|
+
step.progress
|
64
|
+
# when a batch of jobs for a step succeeds, we begin processing the `AcidicJob::Run` record again
|
65
|
+
process_run(run)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
data/lib/acidic_job/errors.rb
CHANGED
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Extensions
|
7
|
+
module ActionMailer
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
def deliver_acidicly(_options = {})
|
11
|
+
job = ::ActionMailer::MailDeliveryJob
|
12
|
+
|
13
|
+
job_args = [@mailer_class.name, @action.to_s, "deliver_now", @params, *@args]
|
14
|
+
# for Sidekiq, this depends on the Sidekiq::Serialization extension
|
15
|
+
serialized_job = job.new(job_args).serialize
|
16
|
+
|
17
|
+
AcidicJob::Run.create!(
|
18
|
+
staged: true,
|
19
|
+
job_class: job.name,
|
20
|
+
serialized_job: serialized_job,
|
21
|
+
idempotency_key: IdempotencyKey.value_for(serialized_job)
|
22
|
+
)
|
23
|
+
end
|
24
|
+
alias deliver_transactionally deliver_acidicly
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Extensions
|
7
|
+
module ActiveJob
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
concerning :Serialization do
|
11
|
+
class_methods do
|
12
|
+
def serialize_with_arguments(*args, **kwargs)
|
13
|
+
job_or_instantiate(*args, **kwargs).serialize
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def serialize_job(*_args, **_kwargs)
|
18
|
+
serialize
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
class_methods do
|
23
|
+
def perform_acidicly(*args, **kwargs)
|
24
|
+
raise UnsupportedExtension unless defined?(::ActiveJob) && self < ::ActiveJob::Base
|
25
|
+
|
26
|
+
serialized_job = serialize_with_arguments(*args, **kwargs)
|
27
|
+
|
28
|
+
AcidicJob::Run.create!(
|
29
|
+
staged: true,
|
30
|
+
job_class: name,
|
31
|
+
serialized_job: serialized_job,
|
32
|
+
idempotency_key: IdempotencyKey.value_for(serialized_job)
|
33
|
+
)
|
34
|
+
end
|
35
|
+
alias_method :perform_transactionally, :perform_acidicly
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|