back_ops 0.2.0 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +95 -11
- data/lib/back_ops/action.rb +78 -1
- data/lib/back_ops/operation.rb +17 -2
- data/lib/back_ops/version.rb +1 -1
- data/lib/back_ops/worker.rb +60 -47
- data/lib/generators/back_ops/install_generator.rb +7 -6
- data/lib/generators/back_ops/templates/{migration.rb.tt → create_back_ops_tables.rb.tt} +5 -2
- data/lib/generators/back_ops/templates/update_back_ops_tables_v1.rb.tt +11 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2402fd610b98a0c4f967ba0356034598d273245453fb9f548684ab645c761dc0
|
4
|
+
data.tar.gz: 9490e355132e81b74dc894a670171612adef162a02843ce21588c4814a07446f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc66e230494b0fcefa3ebd3d9fe61a85aabc72c7916e4ab2f309d9d796861213cea375a53b1054686d87f445d50b53ea5d6d58d7c79ffccc89e9ef0929d90046
|
7
|
+
data.tar.gz: 1deceefeaf3325cd4edab641f1be6736899c41e1ac209fe4e770c5331143441cc0b76498e7fc61b3aaa17c5b3bd443c90217913d887c5b7b35c85d3efb36b718
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# BackOps
|
2
2
|
|
3
|
-
Back Ops is intended for background processing of jobs that require multiple tasks to be completed. It executes each task in sequence in a separate Sidekiq worker. This allows for jobs to be retryable if failures occur, but completed tasks are not retried.
|
3
|
+
Back Ops is intended for background processing of jobs that require multiple tasks to be completed. It executes each task in sequence in a separate Sidekiq worker. This allows for jobs to be retryable if failures occur, but completed tasks are not retried.
|
4
4
|
|
5
5
|
Progress and error states are tracked in the database, so that that you are always aware of what was processed, and if any task fails, where the failure occured in the process, what the error message is, what the stack trace is, so you know what's happening and you can always retry the job from the failed task.
|
6
6
|
|
@@ -25,7 +25,7 @@ $ gem install back_ops
|
|
25
25
|
Copy the migration from the gem to your application, then run migrations.
|
26
26
|
|
27
27
|
```bash
|
28
|
-
$ rails g back_ops:install
|
28
|
+
$ rails g back_ops:install --skip
|
29
29
|
$ rails db:migrate
|
30
30
|
```
|
31
31
|
|
@@ -40,25 +40,28 @@ module Subscriptions
|
|
40
40
|
def self.call(subscription)
|
41
41
|
BackOps::Worker.perform_async({
|
42
42
|
subscription_id: subscription.id
|
43
|
-
},
|
44
|
-
|
45
|
-
|
46
|
-
|
43
|
+
}, {
|
44
|
+
main: [
|
45
|
+
Subscriptions::Actions::Fulfillment::ChargeCreditCard,
|
46
|
+
Subscriptions::Actions::Fulfillment::SetupSubscription,
|
47
|
+
Subscriptions::Actions::Fulfillment::SendEmailReceipt
|
48
|
+
]
|
49
|
+
})
|
47
50
|
end
|
48
51
|
end
|
49
52
|
end
|
50
53
|
end
|
51
54
|
```
|
52
55
|
|
53
|
-
Each action receives
|
56
|
+
Each action receives an object with access to all global variables as follows.
|
54
57
|
|
55
58
|
```ruby
|
56
59
|
module Subscriptions
|
57
60
|
module Actions
|
58
61
|
module Fulfillment
|
59
62
|
class ChargeCreditCard
|
60
|
-
def self.call(
|
61
|
-
subscription_id =
|
63
|
+
def self.call(action)
|
64
|
+
subscription_id = action.get(:subscription_id)
|
62
65
|
subscription = Subscription.find(subscription_id)
|
63
66
|
# ...
|
64
67
|
end
|
@@ -71,10 +74,91 @@ end
|
|
71
74
|
You now also have full transparency into each operation and can view it in the admin section by invoking the following code.
|
72
75
|
|
73
76
|
```ruby
|
74
|
-
|
75
|
-
|
77
|
+
operation = BackOps::Operation.includes(:actions).
|
78
|
+
where(name: 'Subscriptions::Operations::Fulfillment').
|
79
|
+
globals_contains(subscription_id: subscription.id).
|
80
|
+
first
|
76
81
|
```
|
77
82
|
|
83
|
+
## Branches
|
84
|
+
|
85
|
+
Sometimes you need to step through an operation based on conditions. You can accomplish this by using branches. To set this up, define a set of branches that your process can take up front, as follows:
|
86
|
+
|
87
|
+
```ruby
|
88
|
+
module Subscriptions
|
89
|
+
module Operations
|
90
|
+
class Fullfillment
|
91
|
+
def self.call(subscription)
|
92
|
+
BackOps::Worker.perform_async({
|
93
|
+
subscription_id: subscription.id
|
94
|
+
}, {
|
95
|
+
main: [
|
96
|
+
Subscriptions::Actions::Fulfillment::ChargeCreditCard,
|
97
|
+
Subscriptions::Actions::Fulfillment::SendEmailReceipt
|
98
|
+
],
|
99
|
+
red_subscriptions: [
|
100
|
+
Subscriptions::Actions::Fulfillment::SetupRedSubscription
|
101
|
+
],
|
102
|
+
blue_subscriptions: [
|
103
|
+
Subscriptions::Actions::Fulfillment::SetupBlueSubscription
|
104
|
+
]
|
105
|
+
})
|
106
|
+
end
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
```
|
111
|
+
|
112
|
+
You can then jump to these branches in the code as follows:
|
113
|
+
|
114
|
+
```ruby
|
115
|
+
module Subscriptions
|
116
|
+
module Actions
|
117
|
+
module Fulfillment
|
118
|
+
class ChargeCreditCard
|
119
|
+
def self.call(action)
|
120
|
+
subscription_id = action.get(:subscription_id)
|
121
|
+
subscription = Subscription.find(subscription_id)
|
122
|
+
# ...
|
123
|
+
|
124
|
+
# The following will force the next action
|
125
|
+
# to be the first action defined in the
|
126
|
+
# :red_subscriptions branch.
|
127
|
+
action.jump_to(:red_subscriptions) if subscription.is_red?
|
128
|
+
|
129
|
+
# OR
|
130
|
+
|
131
|
+
# The following will force the next action
|
132
|
+
# to be the specified action defined in the
|
133
|
+
# :blue_subscriptions branch.
|
134
|
+
action.jump_to(blue_subscriptions: Subscriptions::Actions::Fulfillment::SetupBlueSubscription) if subscription.is_blue?
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
|
141
|
+
# When you're done, jump back to the main branch as follows
|
142
|
+
|
143
|
+
module Subscriptions
|
144
|
+
module Actions
|
145
|
+
module Fulfillment
|
146
|
+
class SetupBlueSubscription
|
147
|
+
def self.call(action)
|
148
|
+
subscription_id = action.get(:subscription_id)
|
149
|
+
subscription = Subscription.find(subscription_id)
|
150
|
+
# ...
|
151
|
+
|
152
|
+
action.jump_to(main: Subscriptions::Actions::Fulfillment::SendEmailReceipt)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
```
|
159
|
+
|
160
|
+
**NOTE:** Jump does not stop the rest of the action from being processed. It merely sets a pointer to the next action to be processed when the current action is complete. To exit out of an action, simply `return`.
|
161
|
+
|
78
162
|
|
79
163
|
## Contributing
|
80
164
|
To contribute to this project. Clone the repo and create a branch for your changes. When complete, create a pull request into the `develop` branch of this project. Ensure that positive and negative test cases cover your changes and all tests pass.
|
data/lib/back_ops/action.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
module BackOps
|
2
2
|
class Action < ActiveRecord::Base
|
3
|
-
|
3
|
+
|
4
4
|
# == Constants ============================================================
|
5
5
|
|
6
6
|
# == Attributes ===========================================================
|
@@ -21,6 +21,83 @@ module BackOps
|
|
21
21
|
|
22
22
|
self.table_name = 'back_ops_actions'
|
23
23
|
|
24
|
+
def self.after(action)
|
25
|
+
next_on_branch = BackOps::Action.where(operation: action.operation, branch: action.branch).
|
26
|
+
where('back_ops_actions.order > ?', action.order).
|
27
|
+
order(order: :asc).
|
28
|
+
limit(1).
|
29
|
+
first
|
30
|
+
|
31
|
+
return next_on_branch if next_on_branch.present?
|
32
|
+
|
33
|
+
BackOps::Action.where({
|
34
|
+
operation: action.operation,
|
35
|
+
branch: 'main',
|
36
|
+
completed_at: nil
|
37
|
+
}).
|
38
|
+
limit(1).
|
39
|
+
first
|
40
|
+
end
|
41
|
+
|
24
42
|
# == Instance Methods =====================================================
|
43
|
+
|
44
|
+
def premature?
|
45
|
+
perform_at.present? && perform_at > Time.zone.now
|
46
|
+
end
|
47
|
+
|
48
|
+
def get(field)
|
49
|
+
self.operation.get(field)
|
50
|
+
end
|
51
|
+
|
52
|
+
def set(field, value)
|
53
|
+
self.operation.set(field, value)
|
54
|
+
end
|
55
|
+
|
56
|
+
def jump_to(pointer)
|
57
|
+
# :branch
|
58
|
+
# { branch: Action }
|
59
|
+
next_action = nil
|
60
|
+
|
61
|
+
if pointer.is_a?(Symbol)
|
62
|
+
next_action = self.operation.actions.
|
63
|
+
where(branch: pointer).
|
64
|
+
order(order: :asc).
|
65
|
+
limit(1).
|
66
|
+
first
|
67
|
+
elsif pointer.is_a?(Hash)
|
68
|
+
branch, name = pointer.first
|
69
|
+
next_action = self.operation.actions.
|
70
|
+
where(name: name, branch: branch).
|
71
|
+
order(order: :asc).
|
72
|
+
limit(1).
|
73
|
+
first
|
74
|
+
else
|
75
|
+
raise ArgumentError, 'jump_to only accepts as Symbol or a Hash'
|
76
|
+
end
|
77
|
+
|
78
|
+
raise "Could not jump_to(#{pointer.inspect}). Action not found." if next_action.nil?
|
79
|
+
|
80
|
+
self.operation.next_action = next_action
|
81
|
+
self.operation.save!
|
82
|
+
end
|
83
|
+
|
84
|
+
def mark_errored(e)
|
85
|
+
self.error_message = e.message
|
86
|
+
self.stack_trace = e.backtrace
|
87
|
+
self.errored_at = Time.zone.now
|
88
|
+
|
89
|
+
self.attempts_count += 1
|
90
|
+
self.save!
|
91
|
+
end
|
92
|
+
|
93
|
+
def mark_completed
|
94
|
+
self.errored_at = nil
|
95
|
+
self.error_message = nil
|
96
|
+
self.stack_trace = nil
|
97
|
+
|
98
|
+
self.completed_at = Time.zone.now
|
99
|
+
self.attempts_count += 1
|
100
|
+
self.save!
|
101
|
+
end
|
25
102
|
end
|
26
103
|
end
|
data/lib/back_ops/operation.rb
CHANGED
@@ -9,12 +9,18 @@ module BackOps
|
|
9
9
|
|
10
10
|
# == Relationships ========================================================
|
11
11
|
|
12
|
+
belongs_to :next_action, class_name: 'BackOps::Action'
|
13
|
+
|
12
14
|
has_many :actions, class_name: 'BackOps::Action'
|
13
15
|
|
14
16
|
# == Validations ==========================================================
|
15
17
|
|
16
18
|
# == Scopes ===============================================================
|
17
19
|
|
20
|
+
scope :globals_contains, ->(hash) {
|
21
|
+
where('globals @> ?', hash.to_json)
|
22
|
+
}
|
23
|
+
|
18
24
|
# == Callbacks ============================================================
|
19
25
|
|
20
26
|
# == Class Methods ========================================================
|
@@ -23,13 +29,22 @@ module BackOps
|
|
23
29
|
|
24
30
|
# == Instance Methods =====================================================
|
25
31
|
|
32
|
+
def first_action
|
33
|
+
self.actions.
|
34
|
+
where(back_ops_actions: { branch: 'main' }).
|
35
|
+
order(order: :asc).
|
36
|
+
limit(1).
|
37
|
+
first
|
38
|
+
end
|
39
|
+
|
26
40
|
def get(field)
|
27
|
-
|
41
|
+
globals[field.to_s]
|
28
42
|
end
|
29
43
|
|
30
44
|
def set(field, value)
|
31
|
-
|
45
|
+
globals[field.to_s] = value
|
32
46
|
save!
|
33
47
|
end
|
48
|
+
|
34
49
|
end
|
35
50
|
end
|
data/lib/back_ops/version.rb
CHANGED
data/lib/back_ops/worker.rb
CHANGED
@@ -13,39 +13,58 @@ module BackOps
|
|
13
13
|
|
14
14
|
# == Class Methods ========================================================
|
15
15
|
|
16
|
-
def self.perform_async(
|
17
|
-
operation = setup_operation_and_actions(
|
16
|
+
def self.perform_async(globals, actions)
|
17
|
+
operation = setup_operation_and_actions(globals, actions)
|
18
18
|
super(operation.id)
|
19
19
|
end
|
20
20
|
|
21
|
-
def self.perform_in(interval,
|
22
|
-
operation = setup_operation_and_actions(
|
21
|
+
def self.perform_in(interval, globals, actions)
|
22
|
+
operation = setup_operation_and_actions(globals, actions)
|
23
23
|
super(interval, operation.id)
|
24
24
|
end
|
25
25
|
|
26
|
-
def self.perform_at(interval,
|
27
|
-
perform_in(interval,
|
26
|
+
def self.perform_at(interval, globals, action)
|
27
|
+
perform_in(interval, globals, action)
|
28
28
|
end
|
29
29
|
|
30
|
-
def self.setup_operation_and_actions(
|
31
|
-
raise ArgumentError, 'Cannot process empty actions' if
|
32
|
-
|
30
|
+
def self.setup_operation_and_actions(globals, branches)
|
31
|
+
raise ArgumentError, 'Cannot process empty actions' if branches.blank?
|
32
|
+
|
33
|
+
globals ||= {}
|
34
|
+
globals.deep_stringify_keys!
|
33
35
|
|
34
36
|
operation = BackOps::Operation.create_or_find_by({
|
35
|
-
params_hash: Digest::MD5.hexdigest("#{
|
37
|
+
params_hash: Digest::MD5.hexdigest("#{globals}|#{branches}"),
|
36
38
|
name: ancestors[1]
|
37
39
|
})
|
38
|
-
operation.
|
40
|
+
operation.globals.merge!(globals)
|
39
41
|
operation.save!
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
42
|
+
|
43
|
+
counter = 0
|
44
|
+
|
45
|
+
branches.each do |branch, actions|
|
46
|
+
actions.each do |action_with_options|
|
47
|
+
action_name, options = [*action_with_options]
|
48
|
+
|
49
|
+
options = {
|
50
|
+
'perform_at' => nil
|
51
|
+
}.merge(options.try(:deep_stringify_keys) || {})
|
52
|
+
|
53
|
+
action = BackOps::Action.create_or_find_by({
|
54
|
+
operation: operation,
|
55
|
+
branch: branch,
|
56
|
+
name: action_name,
|
57
|
+
perform_at: options['perform_at'],
|
58
|
+
order: counter
|
59
|
+
})
|
60
|
+
|
61
|
+
counter += 1
|
62
|
+
end
|
47
63
|
end
|
48
64
|
|
65
|
+
operation.next_action = operation.first_action
|
66
|
+
operation.save!
|
67
|
+
|
49
68
|
operation
|
50
69
|
end
|
51
70
|
|
@@ -59,45 +78,39 @@ module BackOps
|
|
59
78
|
private
|
60
79
|
|
61
80
|
def process(operation)
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
}).order(order: :asc)
|
81
|
+
action = operation.next_action
|
82
|
+
return true if action.blank?
|
83
|
+
return process_next(operation, at: action.perform_at.to_f) if action.premature?
|
66
84
|
|
67
|
-
return true if action_items.blank?
|
68
|
-
|
69
|
-
active_item = action_items[0]
|
70
|
-
next_item = action_items[1]
|
71
|
-
|
72
|
-
if active_item.errored_at.present?
|
73
|
-
active_item.errored_at = nil
|
74
|
-
active_item.error_message = nil
|
75
|
-
active_item.stack_trace = nil
|
76
|
-
active_item.save!
|
77
|
-
end
|
78
|
-
|
79
85
|
begin
|
80
|
-
|
86
|
+
action.name.constantize.call(action)
|
87
|
+
action.mark_completed
|
81
88
|
|
82
|
-
|
83
|
-
|
84
|
-
active_item.save!
|
89
|
+
operation.next_action = BackOps::Action.after(action)
|
90
|
+
operation.save!
|
85
91
|
|
86
|
-
if
|
87
|
-
|
92
|
+
if operation.next_action.present?
|
93
|
+
process_next(operation)
|
88
94
|
else
|
89
|
-
operation.completed_at =
|
95
|
+
operation.completed_at = action.completed_at
|
90
96
|
operation.save!
|
91
97
|
end
|
92
|
-
rescue => e
|
93
|
-
active_item.error_message = e.message
|
94
|
-
active_item.stack_trace = e.backtrace
|
95
|
-
active_item.errored_at = Time.zone.now
|
96
|
-
active_item.attempts_count += 1
|
97
|
-
active_item.save!
|
98
98
|
|
99
|
+
rescue => e
|
100
|
+
action.mark_errored(e)
|
99
101
|
raise
|
100
102
|
end
|
101
103
|
end
|
104
|
+
|
105
|
+
def process_next(operation, options = {})
|
106
|
+
options.deep_stringify_keys!
|
107
|
+
|
108
|
+
Sidekiq::Client.push({
|
109
|
+
'class' => self.class.name,
|
110
|
+
'args' => [operation.id]
|
111
|
+
}.merge(options))
|
112
|
+
|
113
|
+
true
|
114
|
+
end
|
102
115
|
end
|
103
116
|
end
|
@@ -9,18 +9,19 @@ module BackOps
|
|
9
9
|
source_root File.expand_path('templates', __dir__)
|
10
10
|
desc 'Installs the BackOps migration file.'
|
11
11
|
|
12
|
-
def
|
12
|
+
def create_migrations
|
13
13
|
migration_template(
|
14
|
-
'
|
14
|
+
'create_back_ops_tables.rb',
|
15
15
|
'db/migrate/create_back_ops_tables.rb',
|
16
16
|
migration_version: migration_version,
|
17
17
|
)
|
18
|
+
migration_template(
|
19
|
+
'update_back_ops_tables_v1.rb',
|
20
|
+
'db/migrate/update_back_ops_tables_v1.rb',
|
21
|
+
migration_version: migration_version,
|
22
|
+
)
|
18
23
|
end
|
19
24
|
|
20
|
-
# def self.next_migration_number(dirname)
|
21
|
-
# ActiveRecord::Migration.next_migration_number(dirname)
|
22
|
-
# end
|
23
|
-
|
24
25
|
private
|
25
26
|
|
26
27
|
def migration_version
|
@@ -5,18 +5,21 @@ class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version
|
|
5
5
|
create_table :back_ops_operations do |t|
|
6
6
|
t.string :name
|
7
7
|
t.string :params_hash
|
8
|
-
t.jsonb :
|
8
|
+
t.jsonb :globals, null: false, default: {}
|
9
|
+
t.integer :next_action_id, limit: 8
|
9
10
|
t.timestamp :completed_at
|
10
11
|
|
11
12
|
t.timestamps
|
12
13
|
end
|
13
14
|
|
14
15
|
add_index :back_ops_operations, [:name, :params_hash]
|
15
|
-
|
16
|
+
|
16
17
|
create_table :back_ops_actions do |t|
|
17
18
|
t.integer :operation_id, limit: 8
|
18
19
|
t.integer :order, null: false, default: 0
|
20
|
+
t.text :branch
|
19
21
|
t.text :name
|
22
|
+
t.timestamp :perform_at
|
20
23
|
t.text :error_message
|
21
24
|
t.text :stack_trace
|
22
25
|
t.timestamp :errored_at
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
class <%= migration_class_name %> < ActiveRecord::Migration<%= migration_version %>
|
4
|
+
def change
|
5
|
+
add_column :back_ops_operations, :next_action_id, :integer, limit: 8
|
6
|
+
rename_column :back_ops_operations, :context, :globals
|
7
|
+
|
8
|
+
add_column :back_ops_actions, :branch, :text
|
9
|
+
add_column :back_ops_actions, :perform_at, :timestamp
|
10
|
+
end
|
11
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: back_ops
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Aaron Price
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-04-
|
11
|
+
date: 2021-04-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -111,7 +111,8 @@ files:
|
|
111
111
|
- lib/back_ops/version.rb
|
112
112
|
- lib/back_ops/worker.rb
|
113
113
|
- lib/generators/back_ops/install_generator.rb
|
114
|
-
- lib/generators/back_ops/templates/
|
114
|
+
- lib/generators/back_ops/templates/create_back_ops_tables.rb.tt
|
115
|
+
- lib/generators/back_ops/templates/update_back_ops_tables_v1.rb.tt
|
115
116
|
homepage: https://github.com/aaronprice/back_ops
|
116
117
|
licenses:
|
117
118
|
- MIT
|