acidic_job 0.1.5 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +45 -1
- data/README.md +3 -3
- data/acidic_job.gemspec +2 -0
- data/lib/acidic_job.rb +20 -27
- data/lib/acidic_job/no_op.rb +5 -3
- data/lib/acidic_job/recovery_point.rb +9 -7
- data/lib/acidic_job/response.rb +8 -6
- data/lib/acidic_job/version.rb +1 -1
- data/lib/generators/acidic_job_generator.rb +37 -0
- data/lib/generators/templates/migration.rb +17 -0
- metadata +32 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bcdd5a6f2496a5764d0d0b07f1263653a299d2dd2594ac051393278466f59bd2
|
4
|
+
data.tar.gz: 25de84d0345eb47d2c1866b0f7e6824278214aeac1e212adad3504d9682c70a8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 64ddc179ec70f82ebce87438daad5d50a199daee773fd453f7dff27b21c26af5f7a35fe96f7167405b5a24eb275af6e9519cbe53018194ad9388ae0e2c276412
|
7
|
+
data.tar.gz: a960d42269f80fedcd3c913ce7398a8c456759c07c58710533476f3648e38acde98921bdc93a4d26be8fa4405852d20ffbd0685802c114d87e5b9a6650941f59
|
data/Gemfile.lock
CHANGED
@@ -1,12 +1,26 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
acidic_job (0.
|
4
|
+
acidic_job (0.2.0)
|
5
|
+
activerecord (>= 4.0.0)
|
5
6
|
activesupport
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
9
10
|
specs:
|
11
|
+
actionpack (6.1.3.2)
|
12
|
+
actionview (= 6.1.3.2)
|
13
|
+
activesupport (= 6.1.3.2)
|
14
|
+
rack (~> 2.0, >= 2.0.9)
|
15
|
+
rack-test (>= 0.6.3)
|
16
|
+
rails-dom-testing (~> 2.0)
|
17
|
+
rails-html-sanitizer (~> 1.0, >= 1.2.0)
|
18
|
+
actionview (6.1.3.2)
|
19
|
+
activesupport (= 6.1.3.2)
|
20
|
+
builder (~> 3.1)
|
21
|
+
erubi (~> 1.4)
|
22
|
+
rails-dom-testing (~> 2.0)
|
23
|
+
rails-html-sanitizer (~> 1.1, >= 1.2.0)
|
10
24
|
activejob (6.1.3.2)
|
11
25
|
activesupport (= 6.1.3.2)
|
12
26
|
globalid (>= 0.3.6)
|
@@ -22,7 +36,9 @@ GEM
|
|
22
36
|
tzinfo (~> 2.0)
|
23
37
|
zeitwerk (~> 2.3)
|
24
38
|
ast (2.4.2)
|
39
|
+
builder (3.2.4)
|
25
40
|
concurrent-ruby (1.1.9)
|
41
|
+
crass (1.0.6)
|
26
42
|
database_cleaner (2.0.1)
|
27
43
|
database_cleaner-active_record (~> 2.0.0)
|
28
44
|
database_cleaner-active_record (2.0.1)
|
@@ -30,14 +46,40 @@ GEM
|
|
30
46
|
database_cleaner-core (~> 2.0.0)
|
31
47
|
database_cleaner-core (2.0.1)
|
32
48
|
docile (1.4.0)
|
49
|
+
erubi (1.10.0)
|
33
50
|
globalid (0.4.2)
|
34
51
|
activesupport (>= 4.2.0)
|
35
52
|
i18n (1.8.10)
|
36
53
|
concurrent-ruby (~> 1.0)
|
54
|
+
loofah (2.12.0)
|
55
|
+
crass (~> 1.0.2)
|
56
|
+
nokogiri (>= 1.5.9)
|
57
|
+
method_source (1.0.0)
|
58
|
+
mini_portile2 (2.6.1)
|
37
59
|
minitest (5.14.4)
|
60
|
+
nokogiri (1.12.3)
|
61
|
+
mini_portile2 (~> 2.6.1)
|
62
|
+
racc (~> 1.4)
|
63
|
+
nokogiri (1.12.3-x86_64-darwin)
|
64
|
+
racc (~> 1.4)
|
38
65
|
parallel (1.20.1)
|
39
66
|
parser (3.0.1.1)
|
40
67
|
ast (~> 2.4.1)
|
68
|
+
racc (1.5.2)
|
69
|
+
rack (2.2.3)
|
70
|
+
rack-test (1.1.0)
|
71
|
+
rack (>= 1.0, < 3)
|
72
|
+
rails-dom-testing (2.0.3)
|
73
|
+
activesupport (>= 4.2.0)
|
74
|
+
nokogiri (>= 1.6)
|
75
|
+
rails-html-sanitizer (1.4.1)
|
76
|
+
loofah (~> 2.3)
|
77
|
+
railties (6.1.3.2)
|
78
|
+
actionpack (= 6.1.3.2)
|
79
|
+
activesupport (= 6.1.3.2)
|
80
|
+
method_source
|
81
|
+
rake (>= 0.8.7)
|
82
|
+
thor (~> 1.0)
|
41
83
|
rainbow (3.0.0)
|
42
84
|
rake (13.0.4)
|
43
85
|
regexp_parser (2.1.1)
|
@@ -65,6 +107,7 @@ GEM
|
|
65
107
|
simplecov-html (0.12.3)
|
66
108
|
simplecov_json_formatter (0.1.3)
|
67
109
|
sqlite3 (1.4.2)
|
110
|
+
thor (1.1.0)
|
68
111
|
tzinfo (2.0.4)
|
69
112
|
concurrent-ruby (~> 1.0)
|
70
113
|
unicode-display_width (2.0.0)
|
@@ -80,6 +123,7 @@ DEPENDENCIES
|
|
80
123
|
activerecord (~> 6.1.3.2)
|
81
124
|
database_cleaner
|
82
125
|
minitest (~> 5.0)
|
126
|
+
railties (>= 4.0)
|
83
127
|
rake (~> 13.0)
|
84
128
|
rubocop (~> 1.7)
|
85
129
|
rubocop-minitest
|
data/README.md
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
### Idempotent operations for Rails apps, built on top of ActiveJob.
|
4
4
|
|
5
|
-
At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`. With `ActiveJob`, we can easily trigger from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed
|
5
|
+
At the conceptual heart of basically any software are "operations"—the discrete actions the software performs. Rails provides a powerful abstraction layer for building operations in the form of `ActiveJob`. With `ActiveJob`, we can easily trigger from other Ruby code throughout our Rails application (controller actions, model methods, model callbacks, etc.); we can run operations both synchronously (blocking execution and then returning its response to the caller) and asychronously (non-blocking and the caller doesn't know its response); and we can also retry a specific operation if needed seamlessly.
|
6
6
|
|
7
7
|
However, in order to ensure that our operational jobs are _robust_, we need to ensure that they are properly [idempotent and transactional](https://github.com/mperham/sidekiq/wiki/Best-Practices#2-make-your-job-idempotent-and-transactional). As stated in the [GitLab Sidekiq Style Guide](https://docs.gitlab.com/ee/development/sidekiq_style_guide.html#idempotent-jobs):
|
8
8
|
|
@@ -19,7 +19,7 @@ This is, of course, far easier said than done. Thus, `AcidicJob`.
|
|
19
19
|
3. https://brandur.org/job-drain
|
20
20
|
4. https://brandur.org/idempotency-keys
|
21
21
|
|
22
|
-
`
|
22
|
+
`AcidicJob` brings these techniques and principles into the world of a standard Rails application.
|
23
23
|
|
24
24
|
## Installation
|
25
25
|
|
@@ -40,7 +40,7 @@ Or install it yourself as:
|
|
40
40
|
Then, use the following command to copy over the AcidicJobKey migration.
|
41
41
|
|
42
42
|
```
|
43
|
-
rails generate acidic_job
|
43
|
+
rails generate acidic_job
|
44
44
|
```
|
45
45
|
|
46
46
|
## Usage
|
data/acidic_job.gemspec
CHANGED
@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
|
|
28
28
|
spec.require_paths = ["lib"]
|
29
29
|
|
30
30
|
spec.add_dependency "activesupport"
|
31
|
+
spec.add_dependency "activerecord", ">= 4.0.0"
|
32
|
+
spec.add_development_dependency "railties", ">= 4.0"
|
31
33
|
|
32
34
|
# For more information and examples about making a new gem, checkout our
|
33
35
|
# guide at: https://bundler.io/guides/creating_gem.html
|
data/lib/acidic_job.rb
CHANGED
@@ -8,12 +8,6 @@ require "active_support/concern"
|
|
8
8
|
|
9
9
|
# rubocop:disable Metrics/ModuleLength, Style/Documentation, Metrics/AbcSize, Metrics/MethodLength
|
10
10
|
module AcidicJob
|
11
|
-
class IdempotencyKeyRequired < StandardError; end
|
12
|
-
|
13
|
-
class MissingRequiredAttribute < StandardError; end
|
14
|
-
|
15
|
-
class IdempotencyKeyTooShort < StandardError; end
|
16
|
-
|
17
11
|
class MismatchedIdempotencyKeyAndJobArguments < StandardError; end
|
18
12
|
|
19
13
|
class LockedIdempotencyKey < StandardError; end
|
@@ -38,31 +32,32 @@ module AcidicJob
|
|
38
32
|
end
|
39
33
|
|
40
34
|
# Number of seconds passed which we consider a held idempotency key lock to be
|
41
|
-
# defunct and eligible to be locked again by a different
|
35
|
+
# defunct and eligible to be locked again by a different job run. We try to
|
42
36
|
# unlock keys on our various failure conditions, but software is buggy, and
|
43
37
|
# this might not happen 100% of the time, so this is a hedge against it.
|
44
38
|
IDEMPOTENCY_KEY_LOCK_TIMEOUT = 90
|
45
39
|
|
46
|
-
# To try and enforce some level of required randomness in an idempotency key,
|
47
|
-
# we require a minimum length. This of course is a poor approximate, and in
|
48
|
-
# real life you might want to consider trying to measure actual entropy with
|
49
|
-
# something like the Shannon entropy equation.
|
50
|
-
IDEMPOTENCY_KEY_MIN_LENGTH = 20
|
51
|
-
|
52
40
|
# &block
|
53
|
-
def idempotently(with:)
|
41
|
+
def idempotently(with:) # &block
|
54
42
|
# set accessors for each argument passed in to ensure they are available
|
55
43
|
# to the step methods the job will have written
|
56
44
|
define_accessors_for_passed_arguments(with)
|
57
45
|
|
58
46
|
# execute the block to gather the info on what phases are defined for this job
|
59
47
|
defined_steps = yield
|
48
|
+
# [:create_ride_and_audit_record, :create_stripe_charge, :send_receipt]
|
60
49
|
|
61
50
|
# convert the array of steps into a hash of recovery_points and callable actions
|
62
51
|
phases = define_atomic_phases(defined_steps)
|
52
|
+
# { create_ride_and_audit_record: <#Method >, ... }
|
63
53
|
|
64
|
-
# find or create an AcidicJobKey record to store all information about this job
|
54
|
+
# find or create an AcidicJobKey record (our idempotency key) to store all information about this job
|
65
55
|
# side-effect: will set the @key instance variable
|
56
|
+
#
|
57
|
+
# A key concept here is that if two requests try to insert or update within
|
58
|
+
# close proximity, one of the two will be aborted by Postgres because we're
|
59
|
+
# using a transaction with SERIALIZABLE isolation level. It may not look
|
60
|
+
# it, but this code is safe from races.
|
66
61
|
ensure_idempotency_key_record(job_id, with, defined_steps.first)
|
67
62
|
|
68
63
|
# if the key record is already marked as finished, immediately return its result
|
@@ -74,7 +69,7 @@ module AcidicJob
|
|
74
69
|
recovery_point = @key.recovery_point.to_sym
|
75
70
|
|
76
71
|
case recovery_point
|
77
|
-
when
|
72
|
+
when AcidicJobKey::RECOVERY_POINT_FINISHED.to_sym
|
78
73
|
break
|
79
74
|
else
|
80
75
|
raise UnknownRecoveryPoint unless phases.key? recovery_point
|
@@ -122,13 +117,12 @@ module AcidicJob
|
|
122
117
|
end
|
123
118
|
|
124
119
|
def ensure_idempotency_key_record(key_val, job_args, first_step)
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
isolation_level = :read_uncommitted
|
120
|
+
isolation_level = case ActiveRecord::Base.connection.adapter_name.downcase.to_sym
|
121
|
+
when :sqlite
|
122
|
+
:read_uncommitted
|
123
|
+
else
|
124
|
+
:serializable
|
125
|
+
end
|
132
126
|
|
133
127
|
ActiveRecord::Base.transaction(isolation: isolation_level) do
|
134
128
|
@key = AcidicJobKey.find_by(idempotency_key: key_val)
|
@@ -142,8 +136,7 @@ module AcidicJob
|
|
142
136
|
# because the original job was long enough ago.
|
143
137
|
raise LockedIdempotencyKey if @key.locked_at && @key.locked_at > Time.current - IDEMPOTENCY_KEY_LOCK_TIMEOUT
|
144
138
|
|
145
|
-
# Lock the key and update latest run unless the job is already
|
146
|
-
# finished.
|
139
|
+
# Lock the key and update latest run unless the job is already finished.
|
147
140
|
@key.update!(last_run_at: Time.current, locked_at: Time.current) unless @key.finished?
|
148
141
|
else
|
149
142
|
@key = AcidicJobKey.create!(
|
@@ -170,14 +163,14 @@ module AcidicJob
|
|
170
163
|
end
|
171
164
|
|
172
165
|
def define_atomic_phases(defined_steps)
|
173
|
-
defined_steps <<
|
166
|
+
defined_steps << AcidicJobKey::RECOVERY_POINT_FINISHED
|
174
167
|
|
175
168
|
{}.tap do |phases|
|
176
169
|
defined_steps.each_cons(2).map do |enter_method, exit_method|
|
177
170
|
phases[enter_method] = lambda do
|
178
171
|
method(enter_method).call
|
179
172
|
|
180
|
-
if exit_method ==
|
173
|
+
if exit_method.to_s == AcidicJobKey::RECOVERY_POINT_FINISHED
|
181
174
|
Response.new
|
182
175
|
else
|
183
176
|
RecoveryPoint.new(exit_method)
|
data/lib/acidic_job/no_op.rb
CHANGED
@@ -2,14 +2,16 @@
|
|
2
2
|
|
3
3
|
# Represents an action to set a new recovery point. One possible option for a
|
4
4
|
# return from an #atomic_phase block.
|
5
|
-
|
6
|
-
|
5
|
+
module AcidicJob
|
6
|
+
class RecoveryPoint
|
7
|
+
attr_accessor :name
|
7
8
|
|
8
|
-
|
9
|
-
|
10
|
-
|
9
|
+
def initialize(name)
|
10
|
+
self.name = name
|
11
|
+
end
|
11
12
|
|
12
|
-
|
13
|
-
|
13
|
+
def call(key:)
|
14
|
+
key.update_column(:recovery_point, name)
|
15
|
+
end
|
14
16
|
end
|
15
17
|
end
|
data/lib/acidic_job/response.rb
CHANGED
@@ -3,11 +3,13 @@
|
|
3
3
|
# Represents an action to set a new API response (which will be stored onto an
|
4
4
|
# idempotency key). One possible option for a return from an #atomic_phase
|
5
5
|
# block.
|
6
|
-
|
7
|
-
|
8
|
-
key
|
9
|
-
|
10
|
-
|
11
|
-
|
6
|
+
module AcidicJob
|
7
|
+
class Response
|
8
|
+
def call(key:)
|
9
|
+
key.update!(
|
10
|
+
locked_at: nil,
|
11
|
+
recovery_point: AcidicJobKey::RECOVERY_POINT_FINISHED
|
12
|
+
)
|
13
|
+
end
|
12
14
|
end
|
13
15
|
end
|
data/lib/acidic_job/version.rb
CHANGED
@@ -0,0 +1,37 @@
|
|
1
|
+
require "rails/generators"
|
2
|
+
require "rails/generators/active_record"
|
3
|
+
|
4
|
+
# This generator adds a migration for the {FriendlyId::History
|
5
|
+
# FriendlyId::History} addon.
|
6
|
+
class AcidicJobGenerator < ActiveRecord::Generators::Base
|
7
|
+
# ActiveRecord::Generators::Base inherits from Rails::Generators::NamedBase which requires a NAME parameter for the
|
8
|
+
# new table name. Our generator always uses 'acidic_job_keys', so we just set a random name here.
|
9
|
+
argument :name, type: :string, default: "random_name"
|
10
|
+
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
12
|
+
|
13
|
+
def self.next_migration_number(_path)
|
14
|
+
if instance_variable_defined?("@prev_migration_nr")
|
15
|
+
@prev_migration_nr += 1
|
16
|
+
else
|
17
|
+
@prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
|
18
|
+
end
|
19
|
+
|
20
|
+
@prev_migration_nr.to_s
|
21
|
+
end
|
22
|
+
|
23
|
+
# Copies the migration template to db/migrate.
|
24
|
+
def copy_files
|
25
|
+
migration_template "migration.rb",
|
26
|
+
"db/migrate/create_acidid_job_keys.rb"
|
27
|
+
end
|
28
|
+
|
29
|
+
protected
|
30
|
+
def migration_class
|
31
|
+
if ActiveRecord::VERSION::MAJOR >= 5
|
32
|
+
ActiveRecord::Migration["#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}"]
|
33
|
+
else
|
34
|
+
ActiveRecord::Migration
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class CreateAcidicJobKeys < <%= migration_class %>
|
2
|
+
def change
|
3
|
+
create_table :acidic_job_keys do |t|
|
4
|
+
t.string :idempotency_key, null: false
|
5
|
+
t.string :job_name, null: false
|
6
|
+
t.text :job_args, null: false
|
7
|
+
t.datetime :last_run_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
8
|
+
t.datetime :locked_at, null: true
|
9
|
+
t.string :recovery_point, null: false
|
10
|
+
t.text :error_object
|
11
|
+
t.timestamps
|
12
|
+
|
13
|
+
t.index %i[idempotency_key job_name job_args], unique: true,
|
14
|
+
name: "idx_acidic_job_keys_on_idempotency_key_n_job_name_n_job_args"
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: acidic_job
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- fractaledmind
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-08-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -24,6 +24,34 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: activerecord
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 4.0.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 4.0.0
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: railties
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '4.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ">="
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '4.0'
|
27
55
|
description: Idempotent operations for Rails apps, built on top of ActiveJob.
|
28
56
|
email:
|
29
57
|
- stephen.margheim@gmail.com
|
@@ -48,6 +76,8 @@ files:
|
|
48
76
|
- lib/acidic_job/recovery_point.rb
|
49
77
|
- lib/acidic_job/response.rb
|
50
78
|
- lib/acidic_job/version.rb
|
79
|
+
- lib/generators/acidic_job_generator.rb
|
80
|
+
- lib/generators/templates/migration.rb
|
51
81
|
homepage: https://github.com/fractaledmind/acidic_job
|
52
82
|
licenses:
|
53
83
|
- MIT
|