acidic_job 1.0.0.beta.10 → 1.0.0.pre1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/main.yml +12 -36
- data/.gitignore +0 -5
- data/.ruby_version +1 -0
- data/Gemfile +31 -0
- data/Gemfile.lock +130 -136
- data/README.md +58 -278
- data/acidic_job.gemspec +2 -15
- data/bin/console +2 -4
- data/lib/acidic_job/awaiting.rb +68 -0
- data/lib/acidic_job/errors.rb +19 -11
- data/lib/acidic_job/extensions/action_mailer.rb +11 -3
- data/lib/acidic_job/extensions/active_job.rb +39 -0
- data/lib/acidic_job/extensions/noticed.rb +11 -5
- data/lib/acidic_job/extensions/sidekiq.rb +101 -0
- data/lib/acidic_job/finished_point.rb +5 -3
- data/lib/acidic_job/idempotency_key.rb +15 -18
- data/lib/acidic_job/perform_wrapper.rb +36 -9
- data/lib/acidic_job/recovery_point.rb +3 -2
- data/lib/acidic_job/run.rb +42 -268
- data/lib/acidic_job/staging.rb +30 -0
- data/lib/acidic_job/step.rb +83 -0
- data/lib/acidic_job/version.rb +1 -1
- data/lib/acidic_job.rb +244 -20
- data/lib/generators/acidic_job_generator.rb +35 -0
- data/lib/generators/templates/create_acidic_job_runs_migration.rb.erb +19 -0
- metadata +15 -209
- data/.github/FUNDING.yml +0 -13
- data/.tool-versions +0 -1
- data/UPGRADE_GUIDE.md +0 -81
- data/combustion/log/test.log +0 -0
- data/gemfiles/rails_6.1_sidekiq_6.4.gemfile +0 -10
- data/gemfiles/rails_6.1_sidekiq_6.5.gemfile +0 -10
- data/gemfiles/rails_7.0_sidekiq_6.4.gemfile +0 -10
- data/gemfiles/rails_7.0_sidekiq_6.5.gemfile +0 -10
- data/gemfiles/rails_7.1_sidekiq_6.4.gemfile +0 -10
- data/gemfiles/rails_7.1_sidekiq_6.5.gemfile +0 -10
- data/lib/acidic_job/active_kiq.rb +0 -114
- data/lib/acidic_job/arguments.rb +0 -22
- data/lib/acidic_job/base.rb +0 -11
- data/lib/acidic_job/logger.rb +0 -31
- data/lib/acidic_job/mixin.rb +0 -250
- data/lib/acidic_job/processor.rb +0 -95
- data/lib/acidic_job/rails.rb +0 -40
- data/lib/acidic_job/serializer.rb +0 -24
- data/lib/acidic_job/serializers/exception_serializer.rb +0 -41
- data/lib/acidic_job/serializers/finished_point_serializer.rb +0 -24
- data/lib/acidic_job/serializers/job_serializer.rb +0 -27
- data/lib/acidic_job/serializers/range_serializer.rb +0 -28
- data/lib/acidic_job/serializers/recovery_point_serializer.rb +0 -25
- data/lib/acidic_job/serializers/worker_serializer.rb +0 -27
- data/lib/acidic_job/test_case.rb +0 -9
- data/lib/acidic_job/testing.rb +0 -73
- data/lib/acidic_job/workflow.rb +0 -70
- data/lib/acidic_job/workflow_builder.rb +0 -35
- data/lib/acidic_job/workflow_step.rb +0 -103
- data/lib/generators/acidic_job/drop_tables_generator.rb +0 -26
- data/lib/generators/acidic_job/install_generator.rb +0 -27
- data/lib/generators/acidic_job/templates/create_acidic_job_runs_migration.rb.erb +0 -19
- data/lib/generators/acidic_job/templates/drop_acidic_job_keys_migration.rb.erb +0 -27
data/lib/acidic_job/workflow.rb
DELETED
@@ -1,70 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
class Workflow
|
5
|
-
# { "step 1": { does: "step 1", awaits: [], then: "step 2" }, ... }
|
6
|
-
def initialize(run, job, step_result = nil)
|
7
|
-
@run = run
|
8
|
-
@job = job
|
9
|
-
@step_result = step_result
|
10
|
-
end
|
11
|
-
|
12
|
-
def execute_current_step
|
13
|
-
rescued_error = false
|
14
|
-
|
15
|
-
begin
|
16
|
-
run_current_step
|
17
|
-
rescue StandardError => e
|
18
|
-
rescued_error = e
|
19
|
-
raise e
|
20
|
-
ensure
|
21
|
-
if rescued_error
|
22
|
-
begin
|
23
|
-
@run.store_error!(rescued_error)
|
24
|
-
rescue StandardError => e
|
25
|
-
# We're already inside an error condition, so swallow any additional
|
26
|
-
# errors from here and just send them to logs.
|
27
|
-
AcidicJob.logger.error("Failed to unlock AcidicJob::Run #{@run.id} because of #{e}.")
|
28
|
-
end
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
# be sure to return the `step_result`
|
33
|
-
# which is set by `run_current_step`
|
34
|
-
# which runs the (wrapped) current step method
|
35
|
-
@step_result
|
36
|
-
end
|
37
|
-
|
38
|
-
def progress_to_next_step
|
39
|
-
return if @run.current_step_finished?
|
40
|
-
return run_step_result unless @run.next_step_finishes?
|
41
|
-
|
42
|
-
@job.run_callbacks :finish do
|
43
|
-
run_step_result
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
private
|
48
|
-
|
49
|
-
def run_current_step
|
50
|
-
wrapped_method = WorkflowStep.new(run: @run, job: @job).wrapped
|
51
|
-
current_step = @run.current_step_name
|
52
|
-
|
53
|
-
AcidicJob.logger.log_run_event("Executing #{current_step}...", @job, @run)
|
54
|
-
@run.with_lock do
|
55
|
-
@step_result = wrapped_method.call(@run)
|
56
|
-
end
|
57
|
-
AcidicJob.logger.log_run_event("Executed #{current_step}.", @job, @run)
|
58
|
-
end
|
59
|
-
|
60
|
-
def run_step_result
|
61
|
-
next_step = @run.next_step_name
|
62
|
-
|
63
|
-
AcidicJob.logger.log_run_event("Progressing to #{next_step}...", @job, @run)
|
64
|
-
@run.with_lock do
|
65
|
-
@step_result.call(run: @run)
|
66
|
-
end
|
67
|
-
AcidicJob.logger.log_run_event("Progressed to #{next_step}.", @job, @run)
|
68
|
-
end
|
69
|
-
end
|
70
|
-
end
|
@@ -1,35 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
class WorkflowBuilder
|
5
|
-
attr_reader :steps
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
@steps = []
|
9
|
-
end
|
10
|
-
|
11
|
-
def step(method_name, awaits: [], for_each: nil)
|
12
|
-
@steps << {
|
13
|
-
"does" => method_name.to_s,
|
14
|
-
"awaits" => awaits,
|
15
|
-
"for_each" => for_each
|
16
|
-
}
|
17
|
-
|
18
|
-
@steps
|
19
|
-
end
|
20
|
-
alias_method "✅", :step
|
21
|
-
|
22
|
-
def define_workflow
|
23
|
-
# [ { does: "step 1", awaits: [] }, { does: "step 2", awaits: [] }, ... ]
|
24
|
-
@steps << { "does" => Run::FINISHED_RECOVERY_POINT.to_s }
|
25
|
-
|
26
|
-
{}.tap do |workflow|
|
27
|
-
@steps.each_cons(2).map do |enter_step, exit_step|
|
28
|
-
enter_name = enter_step["does"]
|
29
|
-
workflow[enter_name] = enter_step.merge("then" => exit_step["does"])
|
30
|
-
end
|
31
|
-
end
|
32
|
-
# { "step 1": { does: "step 1", awaits: [], then: "step 2" }, ... }
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
@@ -1,103 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module AcidicJob
|
4
|
-
class WorkflowStep
|
5
|
-
def initialize(run:, job:)
|
6
|
-
@run = run
|
7
|
-
@job = job
|
8
|
-
end
|
9
|
-
|
10
|
-
def wrapped
|
11
|
-
# return a callable Proc with a consistent interface for the execution phase
|
12
|
-
proc do |_run|
|
13
|
-
current_step_result = process_current_step
|
14
|
-
|
15
|
-
if current_step_result.is_a?(FinishedPoint)
|
16
|
-
current_step_result
|
17
|
-
elsif next_item.present?
|
18
|
-
@run.attr_accessors[iterated_key] = prev_iterateds + [next_item]
|
19
|
-
@run.save!(validate: false)
|
20
|
-
RecoveryPoint.new(current_step_name)
|
21
|
-
elsif @run.next_step_finishes?
|
22
|
-
FinishedPoint.new
|
23
|
-
else
|
24
|
-
RecoveryPoint.new(@run.next_step_name)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
private
|
30
|
-
|
31
|
-
def process_current_step
|
32
|
-
result = nil
|
33
|
-
|
34
|
-
if iterable_key.present? && next_item.present? # have an item to iterate over, so pass it to the step method
|
35
|
-
result = current_callable.call(next_item)
|
36
|
-
elsif iterable_key.present? && next_item.nil? # have iterated over all items
|
37
|
-
result = true
|
38
|
-
elsif current_callable.arity.zero?
|
39
|
-
result = current_callable.call
|
40
|
-
else
|
41
|
-
raise TooManyParametersForStepMethod
|
42
|
-
end
|
43
|
-
|
44
|
-
result
|
45
|
-
end
|
46
|
-
|
47
|
-
def current_callable
|
48
|
-
return @job.method(current_step_name) if @job.respond_to?(current_step_name, _include_private = true)
|
49
|
-
# jobs can have no-op steps, especially so that they can use only the async/await mechanism for that step
|
50
|
-
return proc {} if @run.current_step_hash["awaits"].present?
|
51
|
-
|
52
|
-
raise UndefinedStepMethod
|
53
|
-
end
|
54
|
-
|
55
|
-
def iterable_key
|
56
|
-
# the `iterable_key` represents the name of the collection accessor
|
57
|
-
# that must be present in `@run.attr_accessors`; that is,
|
58
|
-
# it must have been passed to `persisting` when calling `with_acidic_workflow`
|
59
|
-
for_each = @run.current_step_hash["for_each"]
|
60
|
-
|
61
|
-
return unless for_each.present?
|
62
|
-
|
63
|
-
return for_each if @run.attr_accessors.key?(for_each)
|
64
|
-
|
65
|
-
raise UnknownForEachCollection
|
66
|
-
end
|
67
|
-
|
68
|
-
def iterated_key
|
69
|
-
# in order to ensure we don't iterate over successfully iterated values in previous runs,
|
70
|
-
# we need to store the collection of already processed values.
|
71
|
-
# we store this collection under a key bound to the current step to ensure multiple steps
|
72
|
-
# can iterate over the same collection.
|
73
|
-
"processed_#{current_step_name}_#{iterable_key}"
|
74
|
-
end
|
75
|
-
|
76
|
-
def prev_iterables
|
77
|
-
# The collection of values to iterate over
|
78
|
-
iterables = @run.attr_accessors.fetch(iterable_key, [])
|
79
|
-
|
80
|
-
return Array(iterables) if iterables.is_a?(Enumerable)
|
81
|
-
|
82
|
-
raise UniterableForEachCollection
|
83
|
-
end
|
84
|
-
|
85
|
-
def prev_iterateds
|
86
|
-
# The collection of values already iterated over
|
87
|
-
iterateds = @run.attr_accessors.fetch(iterated_key, [])
|
88
|
-
|
89
|
-
Array(iterateds)
|
90
|
-
end
|
91
|
-
|
92
|
-
def next_item
|
93
|
-
# The collection of values to iterate over now
|
94
|
-
curr_iterables = prev_iterables - prev_iterateds
|
95
|
-
|
96
|
-
curr_iterables.first
|
97
|
-
end
|
98
|
-
|
99
|
-
def current_step_name
|
100
|
-
@run.current_step_name
|
101
|
-
end
|
102
|
-
end
|
103
|
-
end
|
@@ -1,26 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "rails/generators/active_record"
|
4
|
-
|
5
|
-
module AcidicJob
|
6
|
-
module Generators
|
7
|
-
class DropTablesGenerator < ::Rails::Generators::Base
|
8
|
-
include ActiveRecord::Generators::Migration
|
9
|
-
source_root File.expand_path("templates", __dir__)
|
10
|
-
|
11
|
-
desc "Drops the pre-1.0 tables for the AcidicJob::Key and AcidicJob::Staged models."
|
12
|
-
|
13
|
-
def copy_migration
|
14
|
-
migration_template "drop_acidic_job_keys_migration.rb.erb",
|
15
|
-
"db/migrate/drop_old_acidic_job_tables.rb",
|
16
|
-
migration_version: migration_version
|
17
|
-
end
|
18
|
-
|
19
|
-
protected
|
20
|
-
|
21
|
-
def migration_version
|
22
|
-
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
23
|
-
end
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require "rails/generators/active_record"
|
4
|
-
|
5
|
-
module AcidicJob
|
6
|
-
module Generators
|
7
|
-
class InstallGenerator < ::Rails::Generators::Base
|
8
|
-
include ActiveRecord::Generators::Migration
|
9
|
-
source_root File.expand_path("templates", __dir__)
|
10
|
-
|
11
|
-
desc "Generates a migration for the AcidicJob::Run table."
|
12
|
-
|
13
|
-
# Copies the migration template to db/migrate.
|
14
|
-
def copy_acidic_job_runs_migration_files
|
15
|
-
migration_template "create_acidic_job_runs_migration.rb.erb",
|
16
|
-
"db/migrate/create_acidic_job_runs.rb",
|
17
|
-
migration_version: migration_version
|
18
|
-
end
|
19
|
-
|
20
|
-
protected
|
21
|
-
|
22
|
-
def migration_version
|
23
|
-
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
-
def change
|
3
|
-
create_table :acidic_job_runs do |t|
|
4
|
-
t.boolean :staged, null: false, default: -> { false }
|
5
|
-
t.string :idempotency_key, null: false, index: { unique: true }
|
6
|
-
t.text :serialized_job, null: false
|
7
|
-
t.string :job_class, null: false
|
8
|
-
t.datetime :last_run_at, null: true, default: -> { "CURRENT_TIMESTAMP" }
|
9
|
-
t.datetime :locked_at, null: true
|
10
|
-
t.string :recovery_point, null: true
|
11
|
-
t.text :error_object, null: true
|
12
|
-
t.text :attr_accessors, null: true
|
13
|
-
t.text :workflow, null: true
|
14
|
-
t.references :awaited_by, null: true, index: true
|
15
|
-
t.text :returning_to, null: true
|
16
|
-
t.timestamps
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
@@ -1,27 +0,0 @@
|
|
1
|
-
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
2
|
-
def change
|
3
|
-
drop_table :acidic_job_keys do |t|
|
4
|
-
t.string :idempotency_key, null: false
|
5
|
-
t.string :job_name, null: false
|
6
|
-
t.text :job_args, null: true
|
7
|
-
t.datetime :last_run_at, null: false, default: -> { "CURRENT_TIMESTAMP" }
|
8
|
-
t.datetime :locked_at, null: true
|
9
|
-
t.string :recovery_point, null: false
|
10
|
-
t.text :error_object
|
11
|
-
t.text :attr_accessors
|
12
|
-
t.text :workflow
|
13
|
-
t.timestamps
|
14
|
-
|
15
|
-
t.index %i[idempotency_key job_name job_args],
|
16
|
-
unique: true,
|
17
|
-
name: "idx_acidic_job_keys_on_idempotency_key_n_job_name_n_job_args"
|
18
|
-
end
|
19
|
-
|
20
|
-
drop_table :staged_acidic_jobs do |t|
|
21
|
-
t.string :adapter, null: false
|
22
|
-
t.string :job_name, null: false
|
23
|
-
t.text :job_args, null: true
|
24
|
-
t.timestamps
|
25
|
-
end
|
26
|
-
end
|
27
|
-
end
|