acidic_job 0.7.7 → 0.8.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +13 -0
- data/.github/workflows/main.yml +36 -12
- data/.gitignore +6 -0
- data/.rubocop.yml +21 -0
- data/.tool-versions +1 -0
- data/Gemfile +0 -25
- data/Gemfile.lock +176 -74
- data/README.md +299 -46
- data/UPGRADE_GUIDE.md +81 -0
- data/acidic_job.gemspec +15 -2
- data/bin/console +5 -2
- data/combustion/log/test.log +0 -0
- data/gemfiles/rails_6.1_sidekiq_6.4.gemfile +10 -0
- data/gemfiles/rails_6.1_sidekiq_6.5.gemfile +10 -0
- data/gemfiles/rails_7.0_sidekiq_6.4.gemfile +10 -0
- data/gemfiles/rails_7.0_sidekiq_6.5.gemfile +10 -0
- data/gemfiles/rails_7.1_sidekiq_6.4.gemfile +10 -0
- data/gemfiles/rails_7.1_sidekiq_6.5.gemfile +10 -0
- data/lib/acidic_job/active_kiq.rb +114 -0
- data/lib/acidic_job/arguments.rb +22 -0
- data/lib/acidic_job/base.rb +11 -0
- data/lib/acidic_job/errors.rb +11 -17
- data/lib/acidic_job/extensions/action_mailer.rb +19 -0
- data/lib/acidic_job/extensions/noticed.rb +46 -0
- data/lib/acidic_job/{response.rb → finished_point.rb} +5 -7
- data/lib/acidic_job/idempotency_key.rb +27 -0
- data/lib/acidic_job/logger.rb +31 -0
- data/lib/acidic_job/mixin.rb +250 -0
- data/lib/acidic_job/perform_wrapper.rb +14 -27
- data/lib/acidic_job/processor.rb +95 -0
- data/lib/acidic_job/rails.rb +40 -0
- data/lib/acidic_job/recovery_point.rb +4 -5
- data/lib/acidic_job/run.rb +303 -0
- data/lib/acidic_job/serializer.rb +24 -0
- data/lib/acidic_job/serializers/exception_serializer.rb +41 -0
- data/lib/acidic_job/serializers/finished_point_serializer.rb +24 -0
- data/lib/acidic_job/serializers/job_serializer.rb +27 -0
- data/lib/acidic_job/serializers/range_serializer.rb +28 -0
- data/lib/acidic_job/serializers/recovery_point_serializer.rb +25 -0
- data/lib/acidic_job/serializers/worker_serializer.rb +27 -0
- data/lib/acidic_job/test_case.rb +9 -0
- data/lib/acidic_job/testing.rb +73 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job/workflow.rb +70 -0
- data/lib/acidic_job/workflow_builder.rb +35 -0
- data/lib/acidic_job/workflow_step.rb +103 -0
- data/lib/acidic_job.rb +25 -334
- data/lib/generators/acidic_job/drop_tables_generator.rb +26 -0
- data/lib/generators/acidic_job/install_generator.rb +27 -0
- data/lib/generators/acidic_job/templates/create_acidic_job_runs_migration.rb.erb +19 -0
- data/lib/generators/{templates/create_acidic_job_keys_migration.rb.erb → acidic_job/templates/drop_acidic_job_keys_migration.rb.erb} +10 -3
- metadata +214 -18
- data/.ruby_version +0 -1
- data/lib/acidic_job/deliver_transactionally_extension.rb +0 -26
- data/lib/acidic_job/key.rb +0 -33
- data/lib/acidic_job/no_op.rb +0 -11
- data/lib/acidic_job/perform_transactionally_extension.rb +0 -33
- data/lib/acidic_job/sidekiq_callbacks.rb +0 -45
- data/lib/acidic_job/staged.rb +0 -50
- data/lib/generators/acidic_job_generator.rb +0 -44
- data/lib/generators/templates/create_staged_acidic_jobs_migration.rb.erb +0 -10
@@ -0,0 +1,114 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
require "active_support/callbacks"
|
5
|
+
require "active_support/core_ext/module/concerning"
|
6
|
+
require_relative "mixin"
|
7
|
+
|
8
|
+
module AcidicJob
|
9
|
+
class ActiveKiq
|
10
|
+
include ::Sidekiq::Worker
|
11
|
+
include ::Sidekiq::JobUtil
|
12
|
+
include ::ActiveSupport::Callbacks
|
13
|
+
define_callbacks :perform
|
14
|
+
include Mixin
|
15
|
+
|
16
|
+
concerning :Initializing do
|
17
|
+
included do
|
18
|
+
attr_accessor :arguments
|
19
|
+
attr_accessor :job_id
|
20
|
+
attr_accessor :queue_name
|
21
|
+
attr_accessor :sidekiq_options
|
22
|
+
end
|
23
|
+
##
|
24
|
+
# Creates a new job instance.
|
25
|
+
# +args+ are the arguments, if any, that will be passed to the perform method
|
26
|
+
# +opts+ are any options to configure the job
|
27
|
+
def initialize(*arguments)
|
28
|
+
@arguments = arguments
|
29
|
+
@job_id = ::SecureRandom.uuid
|
30
|
+
@sidekiq_options = sidekiq_options_hash || ::Sidekiq.default_job_options
|
31
|
+
@queue_name = @sidekiq_options["queue"]
|
32
|
+
end
|
33
|
+
|
34
|
+
# Sidekiq sets the `jid` when it is processing jobs off of the Redis queue.
|
35
|
+
# We override the job identifier when staging jobs to encode the `Run` record global id.
|
36
|
+
# We also override how "ActiveKiq" instance's expose the job identifier to match ActiveJob.
|
37
|
+
# So, we need to ensure that when `jid=` is called, we set the `job_id` instead.
|
38
|
+
def jid=(value)
|
39
|
+
super
|
40
|
+
@job_id = value
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
concerning :Performing do
|
45
|
+
class_methods do
|
46
|
+
def perform_now(*args, **kwargs)
|
47
|
+
new.perform(*args, **kwargs)
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def perform_now(*args, **kwargs)
|
52
|
+
perform(*args, **kwargs)
|
53
|
+
end
|
54
|
+
|
55
|
+
def enqueue
|
56
|
+
::Sidekiq::Client.push(
|
57
|
+
"class" => self.class,
|
58
|
+
"args" => @arguments,
|
59
|
+
"jid" => @job_id,
|
60
|
+
"queue" => @queue_name
|
61
|
+
)
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
concerning :Serializing do
|
66
|
+
class_methods do
|
67
|
+
def deserialize(job_data)
|
68
|
+
job = job_data["job_class"].constantize.new
|
69
|
+
job.deserialize(job_data)
|
70
|
+
job
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def serialize
|
75
|
+
return @serialize if defined? @serialize
|
76
|
+
|
77
|
+
item = @sidekiq_options.merge("class" => self.class.name, "args" => @arguments || [])
|
78
|
+
worker_hash = normalize_item(item)
|
79
|
+
|
80
|
+
@serialize = {
|
81
|
+
"job_class" => worker_hash["class"],
|
82
|
+
"job_id" => @job_id,
|
83
|
+
"queue_name" => worker_hash["queue"],
|
84
|
+
"arguments" => worker_hash["args"]
|
85
|
+
}.merge(worker_hash.except("class", "jid", "queue", "args"))
|
86
|
+
end
|
87
|
+
|
88
|
+
def deserialize(job_data)
|
89
|
+
self.job_id = job_data["job_id"]
|
90
|
+
self.queue_name = job_data["queue_name"]
|
91
|
+
self.arguments = job_data["arguments"]
|
92
|
+
self
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# Following approach used by ActiveJob
|
97
|
+
# https://github.com/rails/rails/blob/93c9534c9871d4adad4bc33b5edc355672b59c61/activejob/lib/active_job/callbacks.rb
|
98
|
+
concerning :Callbacks do
|
99
|
+
class_methods do
|
100
|
+
def around_perform(*filters, &blk)
|
101
|
+
set_callback(:perform, :around, *filters, &blk)
|
102
|
+
end
|
103
|
+
|
104
|
+
def before_perform(*filters, &blk)
|
105
|
+
set_callback(:perform, :before, *filters, &blk)
|
106
|
+
end
|
107
|
+
|
108
|
+
def after_perform(*filters, &blk)
|
109
|
+
set_callback(:perform, :after, *filters, &blk)
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_job/arguments"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Arguments
|
7
|
+
include ActiveJob::Arguments
|
8
|
+
extend self # rubocop:disable Style/ModuleFunction
|
9
|
+
|
10
|
+
# `ActiveJob` will throw an error if it tries to deserialize a GlobalID record.
|
11
|
+
# However, this isn't the behavior that we want for our custom `ActiveRecord` serializer.
|
12
|
+
# Since `ActiveRecord` does _not_ reset instance record state to its pre-transactional state
|
13
|
+
# on a transaction ROLLBACK, we can have GlobalID entries in a serialized column that point to
|
14
|
+
# non-persisted records. This is ok. We should simply return `nil` for that portion of the
|
15
|
+
# serialized field.
|
16
|
+
def deserialize_global_id(hash)
|
17
|
+
GlobalID::Locator.locate hash[GLOBALID_KEY]
|
18
|
+
rescue ActiveRecord::RecordNotFound
|
19
|
+
nil
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
data/lib/acidic_job/errors.rb
CHANGED
@@ -2,24 +2,18 @@
|
|
2
2
|
|
3
3
|
module AcidicJob
|
4
4
|
class Error < StandardError; end
|
5
|
-
|
6
|
-
class MismatchedIdempotencyKeyAndJobArguments < Error; end
|
7
|
-
|
8
|
-
class LockedIdempotencyKey < Error; end
|
9
|
-
|
5
|
+
class MissingWorkflowBlock < Error; end
|
10
6
|
class UnknownRecoveryPoint < Error; end
|
11
|
-
|
12
|
-
class UnknownAtomicPhaseType < Error; end
|
13
|
-
|
14
|
-
class SerializedTransactionConflict < Error; end
|
15
|
-
|
16
|
-
class UnknownJobAdapter < Error; end
|
17
|
-
|
18
7
|
class NoDefinedSteps < Error; end
|
19
|
-
|
20
|
-
class
|
21
|
-
|
8
|
+
class RedefiningWorkflow < Error; end
|
9
|
+
class UndefinedStepMethod < Error; end
|
10
|
+
class UnknownForEachCollection < Error; end
|
11
|
+
class UniterableForEachCollection < Error; end
|
12
|
+
class UnknownJobAdapter < Error; end
|
13
|
+
class UnknownAwaitedJob < Error; end
|
22
14
|
class TooManyParametersForStepMethod < Error; end
|
23
|
-
|
24
|
-
class
|
15
|
+
class UnserializableValue < Error; end
|
16
|
+
class LockedIdempotencyKey < Error; end
|
17
|
+
class MismatchedIdempotencyKeyAndJobArguments < Error; end
|
18
|
+
class MissingBlockArgument < Error; end
|
25
19
|
end
|
@@ -0,0 +1,19 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Extensions
|
7
|
+
module ActionMailer
|
8
|
+
extend ActiveSupport::Concern
|
9
|
+
|
10
|
+
def deliver_acidicly(_options = {})
|
11
|
+
job_class = ::ActionMailer::MailDeliveryJob
|
12
|
+
job_args = [@mailer_class.name, @action.to_s, "deliver_now", @params, *@args]
|
13
|
+
job = job_class.new(job_args)
|
14
|
+
|
15
|
+
AcidicJob::Run.stage!(job)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module AcidicJob
|
4
|
+
module Extensions
|
5
|
+
module Noticed
|
6
|
+
extend ActiveSupport::Concern
|
7
|
+
|
8
|
+
class_methods do
|
9
|
+
def deliver_acidicly(recipients)
|
10
|
+
new.deliver_acidicly(recipients)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# THIS IS A HACK THAT COPIES AND PASTES KEY PARTS OF THE `Noticed::Base` CODE
|
15
|
+
# IN ORDER TO ALLOW US TO TRANSACTIONALLY DELIVER NOTIFICATIONS
|
16
|
+
# THIS IS THUS LIABLE TO BREAK WHENEVER THAT GEM IS UPDATED
|
17
|
+
def deliver_acidicly(recipients)
|
18
|
+
delivery_methods = self.class.delivery_methods.dup
|
19
|
+
|
20
|
+
Array.wrap(recipients).uniq.each do |recipient|
|
21
|
+
if (index = delivery_methods.find_index { |m| m[:name] == :database })
|
22
|
+
database_delivery_method = delivery_methods.delete_at(index)
|
23
|
+
self.record = run_delivery_method(database_delivery_method,
|
24
|
+
recipient: recipient,
|
25
|
+
enqueue: false,
|
26
|
+
record: nil)
|
27
|
+
end
|
28
|
+
|
29
|
+
delivery_methods.map do |delivery_method|
|
30
|
+
job_class = delivery_method_for(delivery_method[:name], delivery_method[:options])
|
31
|
+
args = {
|
32
|
+
notification_class: self.class.name,
|
33
|
+
options: delivery_method[:options],
|
34
|
+
params: params,
|
35
|
+
recipient: recipient,
|
36
|
+
record: record
|
37
|
+
}
|
38
|
+
job = job_class.new(args)
|
39
|
+
|
40
|
+
AcidicJob::Run.stage!(job)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -1,16 +1,14 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require_relative "run"
|
4
|
+
|
3
5
|
# Represents an action to set a new API response (which will be stored onto an
|
4
6
|
# idempotency key). One possible option for a return from an #atomic_phase
|
5
7
|
# block.
|
6
8
|
module AcidicJob
|
7
|
-
class
|
8
|
-
def call(
|
9
|
-
|
10
|
-
key.update_columns(
|
11
|
-
locked_at: nil,
|
12
|
-
recovery_point: Key::RECOVERY_POINT_FINISHED
|
13
|
-
)
|
9
|
+
class FinishedPoint
|
10
|
+
def call(run:)
|
11
|
+
run.finish!
|
14
12
|
end
|
15
13
|
end
|
16
14
|
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module AcidicJob
|
4
|
+
class IdempotencyKey
|
5
|
+
def initialize(job)
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def value(acidic_by: :job_id)
|
10
|
+
case acidic_by
|
11
|
+
when Proc
|
12
|
+
proc_result = @job.instance_exec(&acidic_by)
|
13
|
+
Digest::SHA1.hexdigest [@job.class.name, proc_result].flatten.join
|
14
|
+
when :job_arguments
|
15
|
+
Digest::SHA1.hexdigest [@job.class.name, @job.arguments].flatten.join
|
16
|
+
else
|
17
|
+
if @job.job_id.start_with? Run::STAGED_JOB_ID_PREFIX
|
18
|
+
# "STG__#{idempotency_key}__#{encoded_global_id}"
|
19
|
+
_prefix, idempotency_key, _encoded_global_id = @job.job_id.split("__")
|
20
|
+
idempotency_key
|
21
|
+
else
|
22
|
+
@job.job_id
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "logger"
|
4
|
+
require "active_support/tagged_logging"
|
5
|
+
|
6
|
+
module AcidicJob
|
7
|
+
class Logger < ::Logger
|
8
|
+
def log_run_event(msg, job = nil, run = nil)
|
9
|
+
tags = [
|
10
|
+
run&.idempotency_key,
|
11
|
+
inspect_name(job)
|
12
|
+
].compact
|
13
|
+
|
14
|
+
tagged(*tags) { debug(msg) }
|
15
|
+
end
|
16
|
+
|
17
|
+
def inspect_name(obj)
|
18
|
+
return if obj.nil?
|
19
|
+
|
20
|
+
obj.inspect.split.first.remove("#<")
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.logger
|
25
|
+
@logger ||= ActiveSupport::TaggedLogging.new(AcidicJob::Logger.new($stdout, level: :debug))
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.silence_logger!
|
29
|
+
@logger = ActiveSupport::TaggedLogging.new(AcidicJob::Logger.new(IO::NULL, level: :debug))
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,250 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "active_support/concern"
|
4
|
+
|
5
|
+
module AcidicJob
|
6
|
+
module Mixin
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
def self.wire_up(other)
|
10
|
+
# Ensure our `perform` method always runs first to gather parameters
|
11
|
+
# and run perform callbacks for Sidekiq workers
|
12
|
+
other.prepend PerformWrapper
|
13
|
+
|
14
|
+
# By default, we unique job runs by the `job_id`
|
15
|
+
other.instance_variable_set(:@acidic_identifier, :job_id)
|
16
|
+
# However, you can customize this behavior on a per job class level
|
17
|
+
other.define_singleton_method(:acidic_by_job_identifier) { @acidic_identifier = :job_id }
|
18
|
+
# You could unique job runs by the arguments passed to the job (e.g. memoization)
|
19
|
+
other.define_singleton_method(:acidic_by_job_arguments) { @acidic_identifier = :job_arguments }
|
20
|
+
# Or, you could unique jobs run by any logic you'd like using a block
|
21
|
+
other.define_singleton_method(:acidic_by) { |&block| @acidic_identifier = block }
|
22
|
+
|
23
|
+
# We add a callback to ensure that staged, non-workflow jobs are "finished" after they are "performed".
|
24
|
+
# This allows us to ensure that we can always inspect whether a run is finished and get correct data
|
25
|
+
other.set_callback :perform, :after, :finish_staged_job, if: -> { was_staged_job? && !was_workflow_job? }
|
26
|
+
# We also allow you to write any of your own callbacks keyed to the "finish" event.
|
27
|
+
# The "finish" event is notably different from the "perform" event for acidic jobs,
|
28
|
+
# as any acidic job can have one or more "perform" event (retries or a resume after awaiting jobs),
|
29
|
+
# but will only ever have one "finish" event (when the run successfully completes)
|
30
|
+
other.define_callbacks :finish
|
31
|
+
end
|
32
|
+
|
33
|
+
included do
|
34
|
+
Mixin.wire_up(self)
|
35
|
+
end
|
36
|
+
|
37
|
+
class_methods do
|
38
|
+
def inherited(subclass)
|
39
|
+
Mixin.wire_up(subclass)
|
40
|
+
super
|
41
|
+
end
|
42
|
+
|
43
|
+
# `perform_now` runs a job synchronously and immediately
|
44
|
+
# `perform_later` runs a job asynchronously and queues it immediately
|
45
|
+
# `perform_acidicly` run a job asynchronously and queues it after a successful database commit
|
46
|
+
def perform_acidicly(*args)
|
47
|
+
job = new(*args)
|
48
|
+
|
49
|
+
Run.stage!(job)
|
50
|
+
end
|
51
|
+
|
52
|
+
# Instantiate an instance of a job ready for serialization
|
53
|
+
def with(...)
|
54
|
+
# New delegation syntax (...) was introduced in Ruby 2.7.
|
55
|
+
# https://www.ruby-lang.org/en/news/2019/12/12/separation-of-positional-and-keyword-arguments-in-ruby-3-0/
|
56
|
+
job = new(...)
|
57
|
+
# force the job to resolve the `queue_name`, so that we don't try to serialize a Proc into ActiveRecord
|
58
|
+
job.queue_name
|
59
|
+
job
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def idempotency_key
|
64
|
+
IdempotencyKey.new(self).value(acidic_by: acidic_identifier)
|
65
|
+
end
|
66
|
+
|
67
|
+
protected
|
68
|
+
|
69
|
+
# Short circuits execution by sending execution right to 'finished'.
|
70
|
+
# So, ends the job "successfully"
|
71
|
+
def safely_finish_acidic_job
|
72
|
+
FinishedPoint.new
|
73
|
+
end
|
74
|
+
|
75
|
+
def with_acidic_workflow(persisting: {}, &block)
|
76
|
+
raise UnknownJobAdapter unless (defined?(::AcidicJob::Base) && self.class < ::AcidicJob::Base) ||
|
77
|
+
(defined?(::AcidicJob::ActiveKiq) && self.class < ::AcidicJob::ActiveKiq)
|
78
|
+
|
79
|
+
raise RedefiningWorkflow if defined? @workflow_builder
|
80
|
+
|
81
|
+
@workflow_builder = WorkflowBuilder.new
|
82
|
+
|
83
|
+
raise MissingWorkflowBlock, "A block must be passed to `with_acidic_workflow`" unless block_given?
|
84
|
+
raise MissingBlockArgument, "An argument must be passed to the `with_acidic_workflow` block" if block.arity.zero?
|
85
|
+
|
86
|
+
block.call @workflow_builder
|
87
|
+
|
88
|
+
raise NoDefinedSteps if @workflow_builder.steps.empty?
|
89
|
+
|
90
|
+
# convert the array of steps into a hash of recovery_points and next steps
|
91
|
+
workflow = @workflow_builder.define_workflow
|
92
|
+
|
93
|
+
ensure_run_record_and_process(workflow, persisting)
|
94
|
+
rescue LocalJumpError
|
95
|
+
raise MissingWorkflowBlock, "A block must be passed to `with_acidic_workflow`"
|
96
|
+
end
|
97
|
+
|
98
|
+
# DEPRECATED
|
99
|
+
# second attempt at the DSL, but `providing` suggested you needed to serialize
|
100
|
+
# any data that would be used in step methods, but simply instance variables work.
|
101
|
+
def with_acidity(providing: {}, &block)
|
102
|
+
::ActiveSupport::Deprecation.new("1.0", "AcidicJob").deprecation_warning(:with_acidity)
|
103
|
+
|
104
|
+
@workflow_builder = WorkflowBuilder.new
|
105
|
+
@workflow_builder.instance_exec(&block)
|
106
|
+
|
107
|
+
raise NoDefinedSteps if @workflow_builder.steps.empty?
|
108
|
+
|
109
|
+
# convert the array of steps into a hash of recovery_points and next steps
|
110
|
+
workflow = @workflow_builder.define_workflow
|
111
|
+
|
112
|
+
ensure_run_record_and_process(workflow, providing)
|
113
|
+
rescue LocalJumpError
|
114
|
+
raise MissingWorkflowBlock, "A block must be passed to `with_acidity`"
|
115
|
+
end
|
116
|
+
|
117
|
+
# DEPRECATED
|
118
|
+
# first attempt at the DSL, but `idempotently` and `with` are too distant from the gem name.
|
119
|
+
def idempotently(with: {}, &block)
|
120
|
+
::ActiveSupport::Deprecation.new("1.0", "AcidicJob").deprecation_warning(:idempotently)
|
121
|
+
|
122
|
+
@workflow_builder = WorkflowBuilder.new
|
123
|
+
@workflow_builder.instance_exec(&block)
|
124
|
+
|
125
|
+
raise NoDefinedSteps if @workflow_builder.steps.empty?
|
126
|
+
|
127
|
+
# convert the array of steps into a hash of recovery_points and next steps
|
128
|
+
workflow = @workflow_builder.define_workflow
|
129
|
+
|
130
|
+
ensure_run_record_and_process(workflow, with)
|
131
|
+
rescue LocalJumpError
|
132
|
+
raise MissingWorkflowBlock, "A block must be passed to `idempotently`"
|
133
|
+
end
|
134
|
+
|
135
|
+
private
|
136
|
+
|
137
|
+
# You can always retrieve the unique identifier from within the job instance itself
|
138
|
+
def acidic_identifier
|
139
|
+
self.class.instance_variable_get(:@acidic_identifier)
|
140
|
+
end
|
141
|
+
|
142
|
+
def ensure_run_record_and_process(workflow, persisting)
|
143
|
+
::AcidicJob.logger.log_run_event("Initializing run...", self, nil)
|
144
|
+
@acidic_job_run = ::ActiveRecord::Base.transaction(isolation: acidic_isolation_level) do
|
145
|
+
run = Run.find_by(idempotency_key: idempotency_key)
|
146
|
+
serialized_job = serialize
|
147
|
+
|
148
|
+
if run.present?
|
149
|
+
# Programs enqueuing multiple jobs with different parameters but the
|
150
|
+
# same idempotency key is a bug.
|
151
|
+
if run.serialized_job["arguments"] != serialized_job["arguments"]
|
152
|
+
raise MismatchedIdempotencyKeyAndJobArguments
|
153
|
+
end
|
154
|
+
|
155
|
+
# Only acquire a lock if the key is unlocked or its lock has expired
|
156
|
+
# because the original job was long enough ago.
|
157
|
+
raise LockedIdempotencyKey if run.lock_active?
|
158
|
+
|
159
|
+
run.update!(
|
160
|
+
last_run_at: Time.current,
|
161
|
+
locked_at: Time.current,
|
162
|
+
# staged workflow jobs won't have the `workflow` or `recovery_point` stored when staged,
|
163
|
+
# so we need to persist both here.
|
164
|
+
workflow: workflow,
|
165
|
+
recovery_point: run.recovery_point || workflow.keys.first
|
166
|
+
)
|
167
|
+
else
|
168
|
+
run = Run.create!(
|
169
|
+
staged: false,
|
170
|
+
idempotency_key: idempotency_key,
|
171
|
+
job_class: self.class.name,
|
172
|
+
locked_at: Time.current,
|
173
|
+
last_run_at: Time.current,
|
174
|
+
workflow: workflow,
|
175
|
+
recovery_point: workflow.keys.first,
|
176
|
+
serialized_job: serialize
|
177
|
+
)
|
178
|
+
end
|
179
|
+
|
180
|
+
# persist `persisting` values and set accessors for each
|
181
|
+
# first, get the current state of all accessors for both previously persisted and initialized values
|
182
|
+
current_accessors = persisting.stringify_keys.merge(run.attr_accessors)
|
183
|
+
|
184
|
+
# next, ensure that `Run#attr_accessors` is populated with initial values
|
185
|
+
# skip validations for this call to ensure a write
|
186
|
+
run.update_column(:attr_accessors, current_accessors) if current_accessors != run.attr_accessors
|
187
|
+
|
188
|
+
# finally, set reader and writer methods
|
189
|
+
current_accessors.each do |accessor, value|
|
190
|
+
# the reader method may already be defined
|
191
|
+
self.class.attr_reader accessor unless respond_to?(accessor)
|
192
|
+
# but we should always update the value to match the current value
|
193
|
+
instance_variable_set("@#{accessor}", value)
|
194
|
+
# and we overwrite the setter to ensure any updates to an accessor update the `Run` stored value
|
195
|
+
# Note: we must define the singleton method on the instance to avoid overwriting setters on other
|
196
|
+
# instances of the same class
|
197
|
+
define_singleton_method("#{accessor}=") do |updated_value|
|
198
|
+
instance_variable_set("@#{accessor}", updated_value)
|
199
|
+
run.attr_accessors[accessor] = updated_value
|
200
|
+
run.save!(validate: false)
|
201
|
+
updated_value
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
# ensure that we return the `run` record so that the result of this block is the record
|
206
|
+
run
|
207
|
+
end
|
208
|
+
::AcidicJob.logger.log_run_event("Initialized run.", self, @acidic_job_run)
|
209
|
+
|
210
|
+
Processor.new(@acidic_job_run, self).process_run
|
211
|
+
end
|
212
|
+
|
213
|
+
def was_staged_job?
|
214
|
+
job_id.start_with? Run::STAGED_JOB_ID_PREFIX
|
215
|
+
end
|
216
|
+
|
217
|
+
def was_workflow_job?
|
218
|
+
return false unless defined? @acidic_job_run
|
219
|
+
|
220
|
+
@acidic_job_run.present?
|
221
|
+
end
|
222
|
+
|
223
|
+
def staged_job_run
|
224
|
+
return unless was_staged_job?
|
225
|
+
# memoize so we don't have to make unnecessary database calls
|
226
|
+
return @staged_job_run if defined? @staged_job_run
|
227
|
+
|
228
|
+
# "STG__#{idempotency_key}__#{encoded_global_id}"
|
229
|
+
_prefix, _idempotency_key, encoded_global_id = job_id.split("__")
|
230
|
+
staged_job_gid = "gid://#{::Base64.urlsafe_decode64(encoded_global_id)}"
|
231
|
+
|
232
|
+
@staged_job_run = ::GlobalID::Locator.locate(staged_job_gid)
|
233
|
+
end
|
234
|
+
|
235
|
+
def finish_staged_job
|
236
|
+
staged_job_run.finish!
|
237
|
+
end
|
238
|
+
|
239
|
+
def acidic_isolation_level
|
240
|
+
case ::ActiveRecord::Base.connection.adapter_name.downcase.to_sym
|
241
|
+
# SQLite doesn't support `serializable` transactions,
|
242
|
+
# so we use the strictest isolation_level is does support
|
243
|
+
when :sqlite
|
244
|
+
:read_uncommitted
|
245
|
+
else
|
246
|
+
:serializable
|
247
|
+
end
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
@@ -1,35 +1,22 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module AcidicJob
|
4
|
+
# NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
|
5
|
+
# WHY?
|
4
6
|
module PerformWrapper
|
5
|
-
def perform(*args
|
6
|
-
|
7
|
-
# so that we can later delete the record in an `after_perform` callback
|
8
|
-
final_arg = args.last
|
9
|
-
if final_arg.is_a?(Hash) && final_arg.key?("staged_job_gid")
|
10
|
-
args = args[0..-2]
|
11
|
-
@staged_job_gid = final_arg["staged_job_gid"]
|
12
|
-
end
|
13
|
-
|
14
|
-
set_arguments_for_perform(*args, **kwargs)
|
15
|
-
|
16
|
-
super(*args, **kwargs)
|
17
|
-
end
|
18
|
-
|
19
|
-
private
|
7
|
+
def perform(*args)
|
8
|
+
@arguments = args
|
20
9
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
[]
|
32
|
-
end
|
10
|
+
# we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
|
11
|
+
if defined?(ActiveJob) && self.class < ActiveJob::Base
|
12
|
+
super(*args)
|
13
|
+
elsif defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
|
14
|
+
run_callbacks :perform do
|
15
|
+
super(*args)
|
16
|
+
end
|
17
|
+
else
|
18
|
+
raise UnknownJobAdapter
|
19
|
+
end
|
33
20
|
end
|
34
21
|
end
|
35
22
|
end
|