acidic_job 1.0.0.pre28 → 1.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/FUNDING.yml +13 -0
- data/.github/workflows/main.yml +12 -15
- data/.gitignore +3 -1
- data/.rubocop.yml +50 -5
- data/.ruby-version +1 -0
- data/Gemfile.lock +114 -198
- data/README.md +163 -246
- data/TODO +77 -0
- data/acidic_job.gemspec +8 -10
- data/app/models/acidic_job/entry.rb +19 -0
- data/app/models/acidic_job/execution.rb +50 -0
- data/app/models/acidic_job/record.rb +11 -0
- data/app/models/acidic_job/value.rb +7 -0
- data/bin/console +5 -2
- data/bin/test_all +26 -0
- data/gemfiles/rails_7.0.gemfile +4 -1
- data/gemfiles/rails_7.1.gemfile +11 -0
- data/gemfiles/rails_7.2.gemfile +11 -0
- data/gemfiles/rails_8.0.gemfile +11 -0
- data/lib/acidic_job/arguments.rb +31 -0
- data/lib/acidic_job/builder.rb +29 -0
- data/lib/acidic_job/context.rb +26 -0
- data/lib/acidic_job/engine.rb +46 -0
- data/lib/acidic_job/errors.rb +91 -12
- data/lib/acidic_job/log_subscriber.rb +50 -0
- data/lib/acidic_job/serializers/exception_serializer.rb +31 -0
- data/lib/acidic_job/serializers/job_serializer.rb +27 -0
- data/lib/acidic_job/serializers/new_record_serializer.rb +25 -0
- data/lib/acidic_job/serializers/range_serializer.rb +28 -0
- data/lib/acidic_job/testing.rb +8 -12
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job/workflow.rb +185 -0
- data/lib/acidic_job.rb +15 -284
- data/lib/generators/acidic_job/install_generator.rb +3 -3
- data/lib/generators/acidic_job/templates/create_acidic_job_tables_migration.rb.erb +33 -0
- metadata +45 -115
- data/.ruby_version +0 -1
- data/.tool-versions +0 -1
- data/gemfiles/rails_6.1.gemfile +0 -8
- data/lib/acidic_job/awaiting.rb +0 -98
- data/lib/acidic_job/extensions/action_mailer.rb +0 -29
- data/lib/acidic_job/extensions/active_job.rb +0 -40
- data/lib/acidic_job/extensions/noticed.rb +0 -54
- data/lib/acidic_job/extensions/sidekiq.rb +0 -111
- data/lib/acidic_job/finished_point.rb +0 -16
- data/lib/acidic_job/idempotency_key.rb +0 -82
- data/lib/acidic_job/perform_wrapper.rb +0 -22
- data/lib/acidic_job/recovery_point.rb +0 -18
- data/lib/acidic_job/rspec_configuration.rb +0 -31
- data/lib/acidic_job/run.rb +0 -100
- data/lib/acidic_job/serializer.rb +0 -163
- data/lib/acidic_job/staging.rb +0 -38
- data/lib/acidic_job/step.rb +0 -104
- data/lib/acidic_job/test_case.rb +0 -9
- data/lib/acidic_job/upgrade_service.rb +0 -118
- data/lib/generators/acidic_job/drop_tables_generator.rb +0 -26
- data/lib/generators/acidic_job/templates/create_acidic_job_runs_migration.rb.erb +0 -19
- data/lib/generators/acidic_job/templates/drop_acidic_job_keys_migration.rb.erb +0 -27
@@ -1,82 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
class IdempotencyKey
|
5
|
-
def initialize(identifier = :job_id)
|
6
|
-
@identifier = identifier
|
7
|
-
end
|
8
|
-
|
9
|
-
def value_for(hash_or_job, *args, **kwargs)
|
10
|
-
value = case @identifier
|
11
|
-
when Proc
|
12
|
-
value_from_proc(hash_or_job, *args, **kwargs)
|
13
|
-
when :job_args
|
14
|
-
value_from_job_args(hash_or_job, *args, **kwargs)
|
15
|
-
else
|
16
|
-
if hash_or_job.is_a?(Hash)
|
17
|
-
value_from_job_id_for_hash(hash_or_job)
|
18
|
-
else
|
19
|
-
value_from_job_id_for_obj(hash_or_job)
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
23
|
-
result = value || value_from_job_args(hash_or_job, *args, **kwargs)
|
24
|
-
|
25
|
-
if result.start_with?("STG__")
|
26
|
-
# "STG__#{idempotency_key}__#{encoded_global_id}"
|
27
|
-
_prefix, idempotency_key, _encoded_global_id = result.split("__")
|
28
|
-
idempotency_key
|
29
|
-
else
|
30
|
-
result
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
private
|
35
|
-
|
36
|
-
def value_from_job_id_for_hash(hash)
|
37
|
-
if hash.key?("job_id")
|
38
|
-
return if hash["job_id"].nil?
|
39
|
-
return if hash["job_id"].empty?
|
40
|
-
|
41
|
-
hash["job_id"]
|
42
|
-
elsif hash.key?("jid")
|
43
|
-
return if hash["jid"].nil?
|
44
|
-
return if hash["jid"].empty?
|
45
|
-
|
46
|
-
hash["jid"]
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
def value_from_job_id_for_obj(obj)
|
51
|
-
if obj.respond_to?(:job_id)
|
52
|
-
return if obj.job_id.nil?
|
53
|
-
return if obj.job_id.empty?
|
54
|
-
|
55
|
-
obj.job_id
|
56
|
-
elsif obj.respond_to?(:jid)
|
57
|
-
return if obj.jid.nil?
|
58
|
-
return if obj.jid.empty?
|
59
|
-
|
60
|
-
obj.jid
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def value_from_job_args(hash_or_job, *args, **kwargs)
|
65
|
-
worker_class = case hash_or_job
|
66
|
-
when Hash
|
67
|
-
hash_or_job["worker"] || hash_or_job["job_class"]
|
68
|
-
else
|
69
|
-
hash_or_job.class.name
|
70
|
-
end
|
71
|
-
|
72
|
-
Digest::SHA1.hexdigest [worker_class, args, kwargs].flatten.join
|
73
|
-
end
|
74
|
-
|
75
|
-
def value_from_proc(_hash_or_job, *args, **kwargs)
|
76
|
-
return if args.empty? && kwargs.empty?
|
77
|
-
|
78
|
-
idempotency_args = Array(@identifier.call(*args, **kwargs))
|
79
|
-
Digest::SHA1.hexdigest idempotency_args.flatten.join
|
80
|
-
end
|
81
|
-
end
|
82
|
-
end
|
@@ -1,22 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
# NOTE: it is essential that this be a bare module and not an ActiveSupport::Concern
|
5
|
-
module PerformWrapper
|
6
|
-
def perform(*args, **kwargs)
|
7
|
-
@__acidic_job_args = args
|
8
|
-
@__acidic_job_kwargs = kwargs
|
9
|
-
|
10
|
-
# we don't want to run the `perform` callbacks twice, since ActiveJob already handles that for us
|
11
|
-
if defined?(ActiveJob) && self.class < ActiveJob::Base
|
12
|
-
super(*args, **kwargs)
|
13
|
-
elsif defined?(Sidekiq) && self.class.include?(Sidekiq::Worker)
|
14
|
-
run_callbacks :perform do
|
15
|
-
super(*args, **kwargs)
|
16
|
-
end
|
17
|
-
else
|
18
|
-
raise UnknownJobAdapter
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
end
|
@@ -1,18 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# Represents an action to set a new recovery point. One possible option for a
|
4
|
-
# return from an #atomic_phase block.
|
5
|
-
module AcidicJob
|
6
|
-
class RecoveryPoint
|
7
|
-
attr_accessor :name
|
8
|
-
|
9
|
-
def initialize(name)
|
10
|
-
@name = name
|
11
|
-
end
|
12
|
-
|
13
|
-
def call(run:)
|
14
|
-
# Skip AR callbacks as there are none on the model
|
15
|
-
run.update_column(:recovery_point, @name)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
@@ -1,31 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "rspec"
|
4
|
-
require "database_cleaner/active_record"
|
5
|
-
|
6
|
-
# see https://github.com/DatabaseCleaner/database_cleaner#how-to-use
|
7
|
-
RSpec.configure do |config|
|
8
|
-
config.use_transactional_fixtures = false
|
9
|
-
|
10
|
-
config.before(:suite) do
|
11
|
-
DatabaseCleaner.clean_with :truncation
|
12
|
-
|
13
|
-
# Here we are defaulting to :transaction but swapping to deletion for some specs;
|
14
|
-
# if your spec or its code-under-test uses
|
15
|
-
# nested transactions then specify :transactional e.g.:
|
16
|
-
# describe "SomeWorker", :transactional do
|
17
|
-
#
|
18
|
-
DatabaseCleaner.strategy = :transaction
|
19
|
-
|
20
|
-
config.before(:context, transactional: true) { DatabaseCleaner.strategy = :deletion }
|
21
|
-
config.after(:context, transactional: true) { DatabaseCleaner.strategy = :transaction }
|
22
|
-
config.before(:context, type: :system) { DatabaseCleaner.strategy = :deletion }
|
23
|
-
config.after(:context, type: :system) { DatabaseCleaner.strategy = :transaction }
|
24
|
-
end
|
25
|
-
|
26
|
-
config.around(:each) do |example|
|
27
|
-
DatabaseCleaner.cleaning do
|
28
|
-
example.run
|
29
|
-
end
|
30
|
-
end
|
31
|
-
end
|
data/lib/acidic_job/run.rb
DELETED
@@ -1,100 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "active_record"
|
4
|
-
require "global_id"
|
5
|
-
require "active_support/core_ext/object/with_options"
|
6
|
-
require_relative "./serializer"
|
7
|
-
|
8
|
-
module AcidicJob
|
9
|
-
class Run < ActiveRecord::Base
|
10
|
-
include GlobalID::Identification
|
11
|
-
|
12
|
-
FINISHED_RECOVERY_POINT = "FINISHED"
|
13
|
-
|
14
|
-
self.table_name = "acidic_job_runs"
|
15
|
-
|
16
|
-
belongs_to :awaited_by, class_name: "AcidicJob::Run", optional: true
|
17
|
-
has_many :batched_runs, class_name: "AcidicJob::Run", foreign_key: "awaited_by_id"
|
18
|
-
|
19
|
-
after_create_commit :enqueue_staged_job, if: :staged?
|
20
|
-
|
21
|
-
serialize :serialized_job, JSON
|
22
|
-
serialize :error_object, Serializer
|
23
|
-
serialize :workflow, Serializer
|
24
|
-
serialize :returning_to, Serializer
|
25
|
-
store :attr_accessors, coder: Serializer
|
26
|
-
|
27
|
-
validates :staged, inclusion: { in: [true, false] } # uses database default
|
28
|
-
validates :serialized_job, presence: true
|
29
|
-
validates :idempotency_key, presence: true, uniqueness: true
|
30
|
-
validates :job_class, presence: true
|
31
|
-
|
32
|
-
scope :staged, -> { where(staged: true) }
|
33
|
-
scope :unstaged, -> { where(staged: false) }
|
34
|
-
scope :finished, -> { where(recovery_point: FINISHED_RECOVERY_POINT) }
|
35
|
-
scope :outstanding, lambda {
|
36
|
-
where.not(recovery_point: FINISHED_RECOVERY_POINT).or(where(recovery_point: [nil, ""]))
|
37
|
-
}
|
38
|
-
|
39
|
-
with_options unless: :staged? do
|
40
|
-
validates :last_run_at, presence: true
|
41
|
-
validates :recovery_point, presence: true
|
42
|
-
validates :workflow, presence: true
|
43
|
-
end
|
44
|
-
|
45
|
-
def self.purge
|
46
|
-
successfully_completed = where(
|
47
|
-
recovery_point: FINISHED_RECOVERY_POINT,
|
48
|
-
error_object: nil
|
49
|
-
)
|
50
|
-
count = successfully_completed.count
|
51
|
-
|
52
|
-
return 0 if count.zero?
|
53
|
-
|
54
|
-
Rails.logger.info("Deleting #{count} successfully completed AcidicJob runs")
|
55
|
-
successfully_completed.delete_all
|
56
|
-
end
|
57
|
-
|
58
|
-
def finished?
|
59
|
-
recovery_point == FINISHED_RECOVERY_POINT
|
60
|
-
end
|
61
|
-
|
62
|
-
def succeeded?
|
63
|
-
finished? && !failed?
|
64
|
-
end
|
65
|
-
|
66
|
-
def failed?
|
67
|
-
error_object.present?
|
68
|
-
end
|
69
|
-
|
70
|
-
def staged_job_id
|
71
|
-
# encode the identifier for this record in the job ID
|
72
|
-
# base64 encoding for minimal security
|
73
|
-
global_id = to_global_id.to_s.remove("gid://")
|
74
|
-
encoded_global_id = Base64.encode64(global_id).strip
|
75
|
-
|
76
|
-
"STG__#{idempotency_key}__#{encoded_global_id}"
|
77
|
-
end
|
78
|
-
|
79
|
-
private
|
80
|
-
|
81
|
-
def enqueue_staged_job
|
82
|
-
return unless staged?
|
83
|
-
|
84
|
-
serialized_staged_job = if serialized_job.key?("jid")
|
85
|
-
serialized_job.merge("jid" => staged_job_id)
|
86
|
-
elsif serialized_job.key?("job_id")
|
87
|
-
serialized_job.merge("job_id" => staged_job_id)
|
88
|
-
else
|
89
|
-
raise UnknownSerializedJobIdentifier
|
90
|
-
end
|
91
|
-
|
92
|
-
job = job_class.constantize.deserialize(serialized_staged_job)
|
93
|
-
|
94
|
-
job.enqueue
|
95
|
-
|
96
|
-
# NOTE: record will be deleted after the job has successfully been performed
|
97
|
-
true
|
98
|
-
end
|
99
|
-
end
|
100
|
-
end
|
@@ -1,163 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "active_job/serializers"
|
4
|
-
require "active_job/arguments"
|
5
|
-
require "json"
|
6
|
-
|
7
|
-
class WorkerSerializer < ActiveJob::Serializers::ObjectSerializer
|
8
|
-
def serialize(worker)
|
9
|
-
super(
|
10
|
-
"class" => worker.class.name,
|
11
|
-
"args" => worker.instance_variable_get(:@__acidic_job_args),
|
12
|
-
"kwargs" => worker.instance_variable_get(:@__acidic_job_kwargs)
|
13
|
-
)
|
14
|
-
end
|
15
|
-
|
16
|
-
def deserialize(hash)
|
17
|
-
worker_class = hash["class"].constantize
|
18
|
-
worker_class.new(*hash["args"], **hash["kwargs"])
|
19
|
-
end
|
20
|
-
|
21
|
-
def serialize?(argument)
|
22
|
-
defined?(::Sidekiq) && argument.class.include?(::Sidekiq::Worker)
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
class JobSerializer < ActiveJob::Serializers::ObjectSerializer
|
27
|
-
def serialize(job)
|
28
|
-
super(job.serialize)
|
29
|
-
end
|
30
|
-
|
31
|
-
def deserialize(hash)
|
32
|
-
job = ActiveJob::Base.deserialize(hash)
|
33
|
-
job.send(:deserialize_arguments_if_needed)
|
34
|
-
if job.arguments.last.is_a?(Hash)
|
35
|
-
*args, kwargs = job.arguments
|
36
|
-
else
|
37
|
-
args = job.arguments
|
38
|
-
kwargs = {}
|
39
|
-
end
|
40
|
-
job.instance_variable_set(:@__acidic_job_args, args)
|
41
|
-
job.instance_variable_set(:@__acidic_job_kwargs, kwargs)
|
42
|
-
|
43
|
-
job
|
44
|
-
end
|
45
|
-
|
46
|
-
def serialize?(argument)
|
47
|
-
defined?(::ActiveJob::Base) && argument.class < ::ActiveJob::Base
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
class ExceptionSerializer < ActiveJob::Serializers::ObjectSerializer
|
52
|
-
def serialize(exception)
|
53
|
-
hash = {
|
54
|
-
"class" => exception.class.name,
|
55
|
-
"message" => exception.message,
|
56
|
-
"cause" => exception.cause,
|
57
|
-
"backtrace" => {}
|
58
|
-
}
|
59
|
-
|
60
|
-
exception.backtrace.map do |trace|
|
61
|
-
path, _, location = trace.rpartition("/")
|
62
|
-
|
63
|
-
next if hash["backtrace"].key?(path)
|
64
|
-
|
65
|
-
hash["backtrace"][path] = location
|
66
|
-
end
|
67
|
-
|
68
|
-
super(hash)
|
69
|
-
end
|
70
|
-
|
71
|
-
def deserialize(hash)
|
72
|
-
exception_class = hash["class"].constantize
|
73
|
-
exception = exception_class.new(hash["message"])
|
74
|
-
exception.set_backtrace(hash["backtrace"].map do |path, location|
|
75
|
-
[path, location].join("/")
|
76
|
-
end)
|
77
|
-
exception
|
78
|
-
end
|
79
|
-
|
80
|
-
def serialize?(argument)
|
81
|
-
defined?(Exception) && argument.is_a?(Exception)
|
82
|
-
end
|
83
|
-
end
|
84
|
-
|
85
|
-
class FinishedPointSerializer < ActiveJob::Serializers::ObjectSerializer
|
86
|
-
def serialize(finished_point)
|
87
|
-
super(
|
88
|
-
"class" => finished_point.class.name
|
89
|
-
)
|
90
|
-
end
|
91
|
-
|
92
|
-
def deserialize(hash)
|
93
|
-
finished_point_class = hash["class"].constantize
|
94
|
-
finished_point_class.new
|
95
|
-
end
|
96
|
-
|
97
|
-
def serialize?(argument)
|
98
|
-
defined?(::AcidicJob::FinishedPoint) && argument.is_a?(::AcidicJob::FinishedPoint)
|
99
|
-
end
|
100
|
-
end
|
101
|
-
|
102
|
-
class RecoveryPointSerializer < ActiveJob::Serializers::ObjectSerializer
|
103
|
-
def serialize(recovery_point)
|
104
|
-
super(
|
105
|
-
"class" => recovery_point.class.name,
|
106
|
-
"name" => recovery_point.name
|
107
|
-
)
|
108
|
-
end
|
109
|
-
|
110
|
-
def deserialize(hash)
|
111
|
-
recovery_point_class = hash["class"].constantize
|
112
|
-
recovery_point_class.new(hash["name"])
|
113
|
-
end
|
114
|
-
|
115
|
-
def serialize?(argument)
|
116
|
-
defined?(::AcidicJob::RecoveryPoint) && argument.is_a?(::AcidicJob::RecoveryPoint)
|
117
|
-
end
|
118
|
-
end
|
119
|
-
|
120
|
-
ActiveJob::Serializers.add_serializers(
|
121
|
-
WorkerSerializer,
|
122
|
-
JobSerializer,
|
123
|
-
ExceptionSerializer,
|
124
|
-
FinishedPointSerializer,
|
125
|
-
RecoveryPointSerializer
|
126
|
-
)
|
127
|
-
|
128
|
-
# ...
|
129
|
-
module AcidicJob
|
130
|
-
module Arguments
|
131
|
-
include ActiveJob::Arguments
|
132
|
-
extend self # rubocop:disable Style/ModuleFunction
|
133
|
-
|
134
|
-
# `ActiveJob` will throw an error if it tries to deserialize a GlobalID record.
|
135
|
-
# However, this isn't the behavior that we want for our custom `ActiveRecord` serializer.
|
136
|
-
# Since `ActiveRecord` does _not_ reset instance record state to its pre-transactional state
|
137
|
-
# on a transaction ROLLBACK, we can have GlobalID entries in a serialized column that point to
|
138
|
-
# non-persisted records. This is ok. We should simply return `nil` for that portion of the
|
139
|
-
# serialized field.
|
140
|
-
def deserialize_global_id(hash)
|
141
|
-
GlobalID::Locator.locate hash[GLOBALID_KEY]
|
142
|
-
rescue ActiveRecord::RecordNotFound
|
143
|
-
nil
|
144
|
-
end
|
145
|
-
end
|
146
|
-
|
147
|
-
class Serializer
|
148
|
-
# Used for `serialize` method in ActiveRecord
|
149
|
-
class << self
|
150
|
-
def load(json)
|
151
|
-
return if json.nil? || json.empty?
|
152
|
-
|
153
|
-
data = JSON.parse(json)
|
154
|
-
Arguments.deserialize(data).first
|
155
|
-
end
|
156
|
-
|
157
|
-
def dump(obj)
|
158
|
-
data = Arguments.serialize [obj]
|
159
|
-
data.to_json
|
160
|
-
end
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
data/lib/acidic_job/staging.rb
DELETED
@@ -1,38 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "active_support/concern"
|
4
|
-
require "global_id/locator"
|
5
|
-
|
6
|
-
module AcidicJob
|
7
|
-
module Staging
|
8
|
-
extend ActiveSupport::Concern
|
9
|
-
|
10
|
-
private
|
11
|
-
|
12
|
-
def was_staged_job?
|
13
|
-
identifier.start_with? "STG__"
|
14
|
-
end
|
15
|
-
|
16
|
-
def staged_job_run
|
17
|
-
# "STG_#{idempotency_key}__#{encoded_global_id}"
|
18
|
-
encoded_global_id = identifier.split("__").last
|
19
|
-
staged_job_gid = "gid://#{Base64.decode64(encoded_global_id)}"
|
20
|
-
|
21
|
-
GlobalID::Locator.locate(staged_job_gid)
|
22
|
-
rescue ActiveRecord::RecordNotFound
|
23
|
-
nil
|
24
|
-
end
|
25
|
-
|
26
|
-
def identifier
|
27
|
-
return jid if defined?(jid) && !jid.nil?
|
28
|
-
return job_id if defined?(job_id) && !job_id.nil?
|
29
|
-
|
30
|
-
# might be defined already in `with_acidity` method
|
31
|
-
acidic_identifier = self.class.acidic_identifier
|
32
|
-
@__acidic_job_idempotency_key ||= IdempotencyKey.new(acidic_identifier)
|
33
|
-
.value_for(self, *@__acidic_job_args, **@__acidic_job_kwargs)
|
34
|
-
|
35
|
-
@__acidic_job_idempotency_key
|
36
|
-
end
|
37
|
-
end
|
38
|
-
end
|
data/lib/acidic_job/step.rb
DELETED
@@ -1,104 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
# Each AcidicJob::Step requires two phases: [1] execution and [2] progression
|
5
|
-
class Step
|
6
|
-
def initialize(step, run, job, step_result = nil)
|
7
|
-
@step = step
|
8
|
-
@run = run
|
9
|
-
@job = job
|
10
|
-
@step_result = step_result
|
11
|
-
end
|
12
|
-
|
13
|
-
# The execution phase performs the work of the defined step
|
14
|
-
def execute
|
15
|
-
rescued_error = false
|
16
|
-
step_callable = wrap_step_as_acidic_callable @step
|
17
|
-
|
18
|
-
begin
|
19
|
-
@run.with_lock do
|
20
|
-
@step_result = step_callable.call(@run)
|
21
|
-
end
|
22
|
-
# QUESTION: Can an error not inherit from StandardError
|
23
|
-
rescue StandardError => e
|
24
|
-
rescued_error = e
|
25
|
-
raise e
|
26
|
-
ensure
|
27
|
-
if rescued_error
|
28
|
-
# If we're leaving under an error condition, try to unlock the job
|
29
|
-
# run right away so that another request can try again.
|
30
|
-
begin
|
31
|
-
@run.update_columns(locked_at: nil, error_object: rescued_error)
|
32
|
-
rescue StandardError => e
|
33
|
-
# We're already inside an error condition, so swallow any additional
|
34
|
-
# errors from here and just send them to logs.
|
35
|
-
# TODO: implement and use a logger here
|
36
|
-
puts "Failed to unlock AcidicJob::Run #{@run.id} because of #{e}."
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
end
|
41
|
-
|
42
|
-
# The progression phase advances the job run state machine onto the next step
|
43
|
-
def progress
|
44
|
-
@run.with_lock do
|
45
|
-
if @step_result.is_a?(FinishedPoint)
|
46
|
-
@job.run_callbacks :finish do
|
47
|
-
@step_result.call(run: @run)
|
48
|
-
end
|
49
|
-
else
|
50
|
-
@step_result.call(run: @run)
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def wrap_step_as_acidic_callable(step)
|
58
|
-
# {"does" => :enqueue_step, "then" => :next_step, "awaits" => [WorkerWithEnqueueStep::FirstWorker]}
|
59
|
-
current_step = step["does"]
|
60
|
-
next_step = step["then"]
|
61
|
-
# to support iteration within steps
|
62
|
-
iterable_key = step["for_each"]
|
63
|
-
iterated_key = "processed_#{current_step}_#{iterable_key}"
|
64
|
-
iterables = @run.attr_accessors.fetch(iterable_key, []) || []
|
65
|
-
iterateds = @run.attr_accessors.fetch(iterated_key, []) || []
|
66
|
-
next_item = iterables.reject { |item| iterateds.include? item }.first
|
67
|
-
|
68
|
-
# jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
|
69
|
-
callable = if @job.respond_to?(current_step, _include_private = true)
|
70
|
-
@job.method(current_step)
|
71
|
-
else
|
72
|
-
proc {}
|
73
|
-
end
|
74
|
-
|
75
|
-
# return a callable Proc with a consistent interface for the execution phase
|
76
|
-
proc do |run|
|
77
|
-
result = if iterable_key.present? && next_item.present?
|
78
|
-
callable.call(next_item)
|
79
|
-
elsif iterable_key.present? && next_item.nil?
|
80
|
-
true
|
81
|
-
elsif callable.arity.zero?
|
82
|
-
callable.call
|
83
|
-
elsif callable.arity == 1
|
84
|
-
callable.call(run)
|
85
|
-
else
|
86
|
-
raise TooManyParametersForStepMethod
|
87
|
-
end
|
88
|
-
|
89
|
-
if result.is_a?(FinishedPoint)
|
90
|
-
result
|
91
|
-
elsif next_item.present?
|
92
|
-
iterateds << next_item
|
93
|
-
@run.attr_accessors[iterated_key] = iterateds
|
94
|
-
@run.save!(validate: false)
|
95
|
-
RecoveryPoint.new(current_step)
|
96
|
-
elsif next_step.to_s == Run::FINISHED_RECOVERY_POINT
|
97
|
-
FinishedPoint.new
|
98
|
-
else
|
99
|
-
RecoveryPoint.new(next_step)
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
104
|
-
end
|
data/lib/acidic_job/test_case.rb
DELETED
@@ -1,118 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "active_support/concern"
|
4
|
-
|
5
|
-
module AcidicJob
|
6
|
-
# recreate the original `Key` model
|
7
|
-
class Key < ::ActiveRecord::Base
|
8
|
-
RECOVERY_POINT_FINISHED = "FINISHED"
|
9
|
-
|
10
|
-
self.table_name = "acidic_job_keys"
|
11
|
-
|
12
|
-
serialize :error_object
|
13
|
-
serialize :job_args
|
14
|
-
serialize :workflow
|
15
|
-
store :attr_accessors
|
16
|
-
end
|
17
|
-
|
18
|
-
# recreate the original `Staged` model
|
19
|
-
class Staged < ActiveRecord::Base
|
20
|
-
self.table_name = "staged_acidic_jobs"
|
21
|
-
|
22
|
-
serialize :job_args
|
23
|
-
|
24
|
-
after_create_commit :enqueue_job
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
def enqueue_job
|
29
|
-
gid = { "staged_job_gid" => to_global_id.to_s }
|
30
|
-
|
31
|
-
if job_args.is_a?(Hash) && job_args.key?("arguments")
|
32
|
-
job_args["arguments"].concat([gid])
|
33
|
-
else
|
34
|
-
job_args.concat([gid])
|
35
|
-
end
|
36
|
-
|
37
|
-
case adapter
|
38
|
-
when "activejob"
|
39
|
-
::ActiveJob::Base.deserialize(job_args).enqueue
|
40
|
-
when "sidekiq"
|
41
|
-
job_name.constantize.perform_async(*job_args)
|
42
|
-
else
|
43
|
-
raise UnknownJobAdapter.new(adapter: adapter)
|
44
|
-
end
|
45
|
-
|
46
|
-
# NOTE: record will be deleted after the job has successfully been performed
|
47
|
-
true
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
module UpgradeService
|
52
|
-
def self.execute
|
53
|
-
# prepare an array to hold the attribute hashes to be passed to `insert_all`
|
54
|
-
run_attributes = []
|
55
|
-
# prepare an array to hold any `Key` records that we couldn't successfully map to `Run` records
|
56
|
-
errored_keys = []
|
57
|
-
|
58
|
-
# iterate over all `AcidicJob::Key` records in batches,
|
59
|
-
# preparing a `Run` attribute hash to be passed to `insert_all`
|
60
|
-
::AcidicJob::Key.find_each do |key|
|
61
|
-
# map all of the simple attributes directly
|
62
|
-
attributes = {
|
63
|
-
id: key.id,
|
64
|
-
staged: false,
|
65
|
-
idempotency_key: key.idempotency_key,
|
66
|
-
job_class: key.job_name,
|
67
|
-
last_run_at: key.last_run_at,
|
68
|
-
locked_at: key.locked_at,
|
69
|
-
recovery_point: key.recovery_point,
|
70
|
-
error_object: key.error_object,
|
71
|
-
attr_accessors: key.attr_accessors,
|
72
|
-
workflow: key.workflow,
|
73
|
-
created_at: key.created_at,
|
74
|
-
updated_at: key.updated_at
|
75
|
-
}
|
76
|
-
|
77
|
-
# prepare the more complicated `job_args` -> `serialized_job` translation
|
78
|
-
job_class = key.job_name.constantize
|
79
|
-
if defined?(::Sidekiq) && job_class.include?(::Sidekiq::Worker)
|
80
|
-
unless job_class.include?(::AcidicJob::Extensions::Sidekiq)
|
81
|
-
job_class.include(::AcidicJob::Extensions::Sidekiq)
|
82
|
-
end
|
83
|
-
job_instance = job_class.new
|
84
|
-
serialized_job = job_instance.serialize_job(*key.job_args)
|
85
|
-
elsif defined?(::ActiveJob) && job_class < ::ActiveJob::Base
|
86
|
-
unless job_class.include?(::AcidicJob::Extensions::ActiveJob)
|
87
|
-
job_class.include(::AcidicJob::Extensions::ActiveJob)
|
88
|
-
end
|
89
|
-
job_args = begin
|
90
|
-
::ActiveJob::Arguments.deserialize(key.job_args)
|
91
|
-
rescue ::ActiveJob::DeserializationError
|
92
|
-
key.job_args
|
93
|
-
end
|
94
|
-
job_instance = job_class.new(*job_args)
|
95
|
-
serialized_job = job_instance.serialize_job
|
96
|
-
end
|
97
|
-
|
98
|
-
attributes[:serialized_job] = serialized_job
|
99
|
-
run_attributes << attributes
|
100
|
-
rescue StandardError => e
|
101
|
-
errored_keys << [e, key]
|
102
|
-
end
|
103
|
-
|
104
|
-
# insert all of the `Run` records
|
105
|
-
::AcidicJob::Run.insert_all(run_attributes)
|
106
|
-
|
107
|
-
# delete all successfully migrated `Key` record
|
108
|
-
::AcidicJob::Key.where(id: ::AcidicJob::Run.select(:id)).delete_all
|
109
|
-
|
110
|
-
# return a report of the upgrade migration
|
111
|
-
{
|
112
|
-
run_records: ::AcidicJob::Run.count,
|
113
|
-
key_records: ::AcidicJob::Key.count,
|
114
|
-
errored_keys: errored_keys
|
115
|
-
}
|
116
|
-
end
|
117
|
-
end
|
118
|
-
end
|