acidic_job 0.7.5 → 1.0.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rubocop.yml +21 -0
- data/Gemfile +6 -0
- data/Gemfile.lock +99 -3
- data/README.md +47 -14
- data/acidic_job.gemspec +2 -2
- data/bin/console +3 -2
- data/lib/acidic_job/awaiting.rb +68 -0
- data/lib/acidic_job/errors.rb +2 -0
- data/lib/acidic_job/extensions/action_mailer.rb +27 -0
- data/lib/acidic_job/extensions/active_job.rb +39 -0
- data/lib/acidic_job/extensions/noticed.rb +52 -0
- data/lib/acidic_job/extensions/sidekiq.rb +101 -0
- data/lib/acidic_job/{response.rb → finished_point.rb} +4 -4
- data/lib/acidic_job/idempotency_key.rb +24 -0
- data/lib/acidic_job/perform_wrapper.rb +34 -20
- data/lib/acidic_job/recovery_point.rb +3 -3
- data/lib/acidic_job/run.rb +77 -0
- data/lib/acidic_job/staging.rb +30 -0
- data/lib/acidic_job/step.rb +83 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +123 -204
- data/lib/generators/acidic_job_generator.rb +6 -15
- data/lib/generators/templates/create_acidic_job_runs_migration.rb.erb +19 -0
- metadata +19 -17
- data/lib/acidic_job/deliver_transactionally_extension.rb +0 -26
- data/lib/acidic_job/key.rb +0 -33
- data/lib/acidic_job/no_op.rb +0 -11
- data/lib/acidic_job/perform_transactionally_extension.rb +0 -33
- data/lib/acidic_job/sidekiq_callbacks.rb +0 -45
- data/lib/acidic_job/staged.rb +0 -50
- data/lib/generators/templates/create_acidic_job_keys_migration.rb.erb +0 -20
- data/lib/generators/templates/create_staged_acidic_jobs_migration.rb.erb +0 -10
@@ -0,0 +1,101 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
require "active_support/callbacks"
|
5
|
+
require "active_support/core_ext/module/concerning"
|
6
|
+
|
7
|
+
module AcidicJob
|
8
|
+
module Extensions
|
9
|
+
module Sidekiq
|
10
|
+
extend ActiveSupport::Concern
|
11
|
+
|
12
|
+
concerning :Serialization do
|
13
|
+
class_methods do
|
14
|
+
# called only from `AcidicJob::Run#enqueue_staged_job`
|
15
|
+
def deserialize(serialized_job_hash)
|
16
|
+
klass = serialized_job_hash["class"].constantize
|
17
|
+
worker = klass.new
|
18
|
+
worker.jid = serialized_job_hash["jid"]
|
19
|
+
worker.instance_variable_set(:@args, serialized_job_hash["args"])
|
20
|
+
|
21
|
+
worker
|
22
|
+
end
|
23
|
+
|
24
|
+
# called only from `AcidicJob::PerformAcidicly#perform_acidicly`
|
25
|
+
# and `AcidicJob::DeliverAcidicly#deliver_acidicly`
|
26
|
+
def serialize_with_arguments(args = [], _kwargs = nil)
|
27
|
+
# THIS IS A HACK THAT ESSENTIALLY COPIES THE CODE FROM THE SIDEKIQ CODEBASE TO MIMIC THE BEHAVIOR
|
28
|
+
args = Array[args]
|
29
|
+
normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(args))
|
30
|
+
item = { "class" => self, "args" => normalized_args }
|
31
|
+
dummy_sidekiq_client = ::Sidekiq::Client.new
|
32
|
+
normed = dummy_sidekiq_client.send :normalize_item, item
|
33
|
+
dummy_sidekiq_client.send :process_single, item["class"], normed
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def serialize_job(args = [], _kwargs = nil)
|
38
|
+
# `@args` is only set via `deserialize`; it is not a standard Sidekiq thing
|
39
|
+
arguments = args || @args
|
40
|
+
normalized_args = ::Sidekiq.load_json(::Sidekiq.dump_json(arguments))
|
41
|
+
item = { "class" => self.class, "args" => normalized_args, "jid" => jid }
|
42
|
+
sidekiq_options = sidekiq_options_hash || {}
|
43
|
+
|
44
|
+
sidekiq_options.merge(item)
|
45
|
+
end
|
46
|
+
|
47
|
+
# called only from `AcidicJob::Run#enqueue_staged_job`
|
48
|
+
def enqueue
|
49
|
+
::Sidekiq::Client.push(
|
50
|
+
"class" => self.class,
|
51
|
+
"args" => @args,
|
52
|
+
"jid" => @jid
|
53
|
+
)
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
concerning :PerformAcidicly do
|
58
|
+
class_methods do
|
59
|
+
def perform_acidicly(*args, **kwargs)
|
60
|
+
serialized_job = serialize_with_arguments(*args, **kwargs)
|
61
|
+
|
62
|
+
AcidicJob::Run.create!(
|
63
|
+
staged: true,
|
64
|
+
job_class: name,
|
65
|
+
serialized_job: serialized_job,
|
66
|
+
idempotency_key: IdempotencyKey.value_for(serialized_job)
|
67
|
+
)
|
68
|
+
end
|
69
|
+
alias_method :perform_transactionally, :perform_acidicly
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
# to balance `perform_async` class method
|
74
|
+
concerning :PerformSync do
|
75
|
+
class_methods do
|
76
|
+
def perform_sync(*args, **kwargs)
|
77
|
+
new.perform(*args, **kwargs)
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
# Following approach used by ActiveJob
|
83
|
+
# https://github.com/rails/rails/blob/93c9534c9871d4adad4bc33b5edc355672b59c61/activejob/lib/active_job/callbacks.rb
|
84
|
+
concerning :Callbacks do
|
85
|
+
class_methods do
|
86
|
+
def around_perform(*filters, &blk)
|
87
|
+
set_callback(:perform, :around, *filters, &blk)
|
88
|
+
end
|
89
|
+
|
90
|
+
def before_perform(*filters, &blk)
|
91
|
+
set_callback(:perform, :before, *filters, &blk)
|
92
|
+
end
|
93
|
+
|
94
|
+
def after_perform(*filters, &blk)
|
95
|
+
set_callback(:perform, :after, *filters, &blk)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
end
|
@@ -4,12 +4,12 @@
|
|
4
4
|
# idempotency key). One possible option for a return from an #atomic_phase
|
5
5
|
# block.
|
6
6
|
module AcidicJob
|
7
|
-
class
|
8
|
-
def call(
|
7
|
+
class FinishedPoint
|
8
|
+
def call(run:)
|
9
9
|
# Skip AR callbacks as there are none on the model
|
10
|
-
|
10
|
+
run.update_columns(
|
11
11
|
locked_at: nil,
|
12
|
-
recovery_point:
|
12
|
+
recovery_point: Run::FINISHED_RECOVERY_POINT
|
13
13
|
)
|
14
14
|
end
|
15
15
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module AcidicJob
|
4
|
+
class IdempotencyKey
|
5
|
+
def self.value_for(hash_or_job, *args, **kwargs)
|
6
|
+
return hash_or_job.job_id if hash_or_job.respond_to?(:job_id) && !hash_or_job.job_id.nil?
|
7
|
+
return hash_or_job.jid if hash_or_job.respond_to?(:jid) && !hash_or_job.jid.nil?
|
8
|
+
|
9
|
+
if hash_or_job.is_a?(Hash) && hash_or_job.key?("job_id") && !hash_or_job["job_id"].nil?
|
10
|
+
return hash_or_job["job_id"]
|
11
|
+
end
|
12
|
+
return hash_or_job["jid"] if hash_or_job.is_a?(Hash) && hash_or_job.key?("jid") && !hash_or_job["jid"].nil?
|
13
|
+
|
14
|
+
worker_class = case hash_or_job
|
15
|
+
when Hash
|
16
|
+
hash_or_job["worker"] || hash_or_job["job_class"]
|
17
|
+
else
|
18
|
+
hash_or_job.class.name
|
19
|
+
end
|
20
|
+
|
21
|
+
Digest::SHA1.hexdigest [worker_class, args, kwargs].flatten.join
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -1,35 +1,49 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module AcidicJob
|
4
|
+
# NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
|
4
5
|
module PerformWrapper
|
5
6
|
def perform(*args, **kwargs)
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
if
|
10
|
-
args
|
11
|
-
|
7
|
+
super_method = method(:perform).super_method
|
8
|
+
|
9
|
+
# we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
|
10
|
+
if aj_job?
|
11
|
+
__acidic_job_perform_for_aj(super_method, *args, **kwargs)
|
12
|
+
elsif sk_job?
|
13
|
+
__acidic_job_perform_for_sk(super_method, *args, **kwargs)
|
14
|
+
else
|
15
|
+
raise UnknownJobAdapter
|
12
16
|
end
|
17
|
+
end
|
13
18
|
|
14
|
-
|
19
|
+
def sk_job?
|
20
|
+
defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
|
21
|
+
end
|
15
22
|
|
16
|
-
|
23
|
+
def aj_job?
|
24
|
+
defined?(ActiveJob) && self.class < ActiveJob::Base
|
17
25
|
end
|
18
26
|
|
19
27
|
private
|
20
28
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
29
|
+
# don't run `perform` callbacks, as ActiveJob already does this
|
30
|
+
def __acidic_job_perform_for_aj(super_method, *args, **kwargs)
|
31
|
+
__acidic_job_perform_base(super_method, *args, **kwargs)
|
32
|
+
end
|
33
|
+
|
34
|
+
# ensure to run `perform` callbacks
|
35
|
+
def __acidic_job_perform_for_sk(super_method, *args, **kwargs)
|
36
|
+
run_callbacks :perform do
|
37
|
+
__acidic_job_perform_base(super_method, *args, **kwargs)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
# capture arguments passed to `perform` to be used by AcidicJob later
|
42
|
+
def __acidic_job_perform_base(super_method, *args, **kwargs)
|
43
|
+
@__acidic_job_args = args
|
44
|
+
@__acidic_job_kwargs = kwargs
|
45
|
+
|
46
|
+
super_method.call(*args, **kwargs)
|
33
47
|
end
|
34
48
|
end
|
35
49
|
end
|
@@ -7,12 +7,12 @@ module AcidicJob
|
|
7
7
|
attr_accessor :name
|
8
8
|
|
9
9
|
def initialize(name)
|
10
|
-
|
10
|
+
@name = name
|
11
11
|
end
|
12
12
|
|
13
|
-
def call(
|
13
|
+
def call(run:)
|
14
14
|
# Skip AR callbacks as there are none on the model
|
15
|
-
|
15
|
+
run.update_column(:recovery_point, @name)
|
16
16
|
end
|
17
17
|
end
|
18
18
|
end
|
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_record"
|
4
|
+
require "global_id"
|
5
|
+
require "active_support/core_ext/object/with_options"
|
6
|
+
|
7
|
+
module AcidicJob
|
8
|
+
class Run < ActiveRecord::Base
|
9
|
+
include GlobalID::Identification
|
10
|
+
|
11
|
+
FINISHED_RECOVERY_POINT = "FINISHED"
|
12
|
+
|
13
|
+
self.table_name = "acidic_job_runs"
|
14
|
+
|
15
|
+
after_create_commit :enqueue_staged_job, if: :staged?
|
16
|
+
|
17
|
+
serialize :error_object
|
18
|
+
serialize :serialized_job
|
19
|
+
serialize :workflow
|
20
|
+
store :attr_accessors
|
21
|
+
|
22
|
+
validates :staged, inclusion: { in: [true, false] } # uses database default
|
23
|
+
validates :serialized_job, presence: true
|
24
|
+
validates :idempotency_key, presence: true, uniqueness: true
|
25
|
+
validates :job_class, presence: true
|
26
|
+
|
27
|
+
scope :staged, -> { where(staged: true) }
|
28
|
+
scope :unstaged, -> { where(staged: false) }
|
29
|
+
scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
|
30
|
+
scope :running, -> { where.not(recovery_point: FINISHED_RECOVERY_POINT) }
|
31
|
+
|
32
|
+
with_options unless: :staged? do
|
33
|
+
validates :last_run_at, presence: true
|
34
|
+
validates :recovery_point, presence: true
|
35
|
+
validates :workflow, presence: true
|
36
|
+
end
|
37
|
+
|
38
|
+
def finished?
|
39
|
+
recovery_point == FINISHED_RECOVERY_POINT
|
40
|
+
end
|
41
|
+
|
42
|
+
def succeeded?
|
43
|
+
finished? && !failed?
|
44
|
+
end
|
45
|
+
|
46
|
+
def failed?
|
47
|
+
error_object.present?
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def enqueue_staged_job
|
53
|
+
return unless staged?
|
54
|
+
|
55
|
+
# encode the identifier for this record in the job ID
|
56
|
+
# base64 encoding for minimal security
|
57
|
+
global_id = to_global_id.to_s.remove("gid://")
|
58
|
+
encoded_global_id = Base64.encode64(global_id).strip
|
59
|
+
staged_job_id = "STG_#{idempotency_key}__#{encoded_global_id}"
|
60
|
+
|
61
|
+
serialized_staged_job = if serialized_job.key?("jid")
|
62
|
+
serialized_job.merge("jid" => staged_job_id)
|
63
|
+
elsif serialized_job.key?("job_id")
|
64
|
+
serialized_job.merge("job_id" => staged_job_id)
|
65
|
+
else
|
66
|
+
raise UnknownSerializedJobIdentifier
|
67
|
+
end
|
68
|
+
|
69
|
+
job = job_class.constantize.deserialize(serialized_staged_job)
|
70
|
+
|
71
|
+
job.enqueue
|
72
|
+
|
73
|
+
# NOTE: record will be deleted after the job has successfully been performed
|
74
|
+
true
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Staging
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
def delete_staged_job_record
|
10
|
+
return unless was_staged_job?
|
11
|
+
|
12
|
+
staged_job_run.delete
|
13
|
+
true
|
14
|
+
rescue ActiveRecord::RecordNotFound
|
15
|
+
true
|
16
|
+
end
|
17
|
+
|
18
|
+
def was_staged_job?
|
19
|
+
identifier.start_with? "STG_"
|
20
|
+
end
|
21
|
+
|
22
|
+
def staged_job_run
|
23
|
+
# "STG_#{idempotency_key}__#{encoded_global_id}"
|
24
|
+
encoded_global_id = identifier.split("__").last
|
25
|
+
staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
|
26
|
+
|
27
|
+
GlobalID::Locator.locate(staged_job_gid)
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module AcidicJob
|
4
|
+
# Each AcidicJob::Step requires two phases: [1] execution and [2] progression
|
5
|
+
class Step
|
6
|
+
def initialize(step, run, job, step_result = nil)
|
7
|
+
@step = step
|
8
|
+
@run = run
|
9
|
+
@job = job
|
10
|
+
@step_result = step_result
|
11
|
+
end
|
12
|
+
|
13
|
+
# The execution phase performs the work of the defined step
|
14
|
+
def execute
|
15
|
+
rescued_error = false
|
16
|
+
step_callable = wrap_step_as_acidic_callable @step
|
17
|
+
|
18
|
+
begin
|
19
|
+
@run.with_lock do
|
20
|
+
@step_result = step_callable.call(@run)
|
21
|
+
end
|
22
|
+
# QUESTION: Can an error not inherit from StandardError
|
23
|
+
rescue StandardError => e
|
24
|
+
rescued_error = e
|
25
|
+
raise e
|
26
|
+
ensure
|
27
|
+
if rescued_error
|
28
|
+
# If we're leaving under an error condition, try to unlock the job
|
29
|
+
# run right away so that another request can try again.
|
30
|
+
begin
|
31
|
+
@run.update_columns(locked_at: nil, error_object: rescued_error)
|
32
|
+
rescue StandardError => e
|
33
|
+
# We're already inside an error condition, so swallow any additional
|
34
|
+
# errors from here and just send them to logs.
|
35
|
+
# TODO: implement and use a logger here
|
36
|
+
puts "Failed to unlock AcidicJob::Run #{@run.id} because of #{e}."
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
# The progression phase advances the job run state machine onto the next step
|
43
|
+
def progress
|
44
|
+
@run.with_lock do
|
45
|
+
@step_result.call(run: @run)
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
private
|
50
|
+
|
51
|
+
def wrap_step_as_acidic_callable(step)
|
52
|
+
# {"does" => :enqueue_step, "then" => :next_step, "awaits" => [WorkerWithEnqueueStep::FirstWorker]}
|
53
|
+
current_step = step["does"]
|
54
|
+
next_step = step["then"]
|
55
|
+
|
56
|
+
# jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
|
57
|
+
callable = if @job.respond_to?(current_step, _include_private = true)
|
58
|
+
@job.method(current_step)
|
59
|
+
else
|
60
|
+
proc {}
|
61
|
+
end
|
62
|
+
|
63
|
+
# return a callable Proc with a consistent interface for the execution phase
|
64
|
+
proc do |run|
|
65
|
+
result = if callable.arity.zero?
|
66
|
+
callable.call
|
67
|
+
elsif callable.arity == 1
|
68
|
+
callable.call(run)
|
69
|
+
else
|
70
|
+
raise TooManyParametersForStepMethod
|
71
|
+
end
|
72
|
+
|
73
|
+
if result.is_a?(FinishedPoint)
|
74
|
+
result
|
75
|
+
elsif next_step.to_s == Run::FINISHED_RECOVERY_POINT
|
76
|
+
FinishedPoint.new
|
77
|
+
else
|
78
|
+
RecoveryPoint.new(next_step)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
data/lib/acidic_job/version.rb
CHANGED