dirty_pipeline 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dirty_pipeline.rb +2 -0
- data/lib/dirty_pipeline/base.rb +52 -85
- data/lib/dirty_pipeline/status.rb +5 -21
- data/lib/dirty_pipeline/storage.rb +34 -4
- data/lib/dirty_pipeline/transaction.rb +76 -0
- data/lib/dirty_pipeline/version.rb +1 -1
- data/lib/dirty_pipeline/worker.rb +20 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8c98084e4a9f05e60dbfb593de8661371a475d63d8831e80873eb0721b9b5fa6
|
4
|
+
data.tar.gz: 445c7d82b17abac6ab4d9c795e1e715bd11ed3bcb944a42e9897ae9c95ef06b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d1cec2be049b6a10eba0629d04287a685ffa6f00af52333647bc342dad67367a3de662b29d6db6e491b51dcbadf75f3d6b118fcf2bf3b5bf601c102fe522d051
|
7
|
+
data.tar.gz: 90ac27df1e8166cc687f76e299350e0a069b983d72c039acd0a4d33881b2b1f9cc728bc0115f3a9af95a0cc781cfba94265838d55ebb8510a90f64e1a4252e0f
|
data/lib/dirty_pipeline.rb
CHANGED
@@ -5,6 +5,8 @@ module DirtyPipeline
|
|
5
5
|
require_relative "dirty_pipeline/storage.rb"
|
6
6
|
require_relative "dirty_pipeline/transition.rb"
|
7
7
|
require_relative "dirty_pipeline/status.rb"
|
8
|
+
require_relative "dirty_pipeline/worker.rb"
|
9
|
+
require_relative "dirty_pipeline/transaction.rb"
|
8
10
|
require_relative "dirty_pipeline/base.rb"
|
9
11
|
|
10
12
|
# This method should yield raw Redis connection
|
data/lib/dirty_pipeline/base.rb
CHANGED
@@ -64,8 +64,16 @@ module DirtyPipeline
|
|
64
64
|
end
|
65
65
|
end
|
66
66
|
|
67
|
+
attr_reader :subject, :error, :storage, :status, :transitions_chain
|
68
|
+
def initialize(subject)
|
69
|
+
@subject = subject
|
70
|
+
@storage = Storage.new(subject, self.class.pipeline_storage)
|
71
|
+
@status = Status.new(self)
|
72
|
+
@transitions_chain = []
|
73
|
+
end
|
74
|
+
|
67
75
|
def enqueue(transition_name, *args)
|
68
|
-
|
76
|
+
DirtyPipeline::Worker.perform_async(
|
69
77
|
"enqueued_pipeline" => self.class.to_s,
|
70
78
|
"find_subject_args" => find_subject_args,
|
71
79
|
"transition_args" => args.unshift(transition_name),
|
@@ -81,18 +89,27 @@ module DirtyPipeline
|
|
81
89
|
end
|
82
90
|
|
83
91
|
def cache
|
84
|
-
storage.
|
92
|
+
storage.last_event["cache"]
|
85
93
|
end
|
86
94
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
95
|
+
def chain(*args)
|
96
|
+
transitions_chain << args
|
97
|
+
self
|
98
|
+
end
|
99
|
+
|
100
|
+
def execute
|
101
|
+
Result() do
|
102
|
+
transitions_chain.each do |targs|
|
103
|
+
call(*targs)
|
104
|
+
storage.increment_transaction_depth!
|
105
|
+
end
|
106
|
+
storage.reset_transaction_depth!
|
107
|
+
transitions_chain.clear
|
108
|
+
end
|
93
109
|
end
|
94
110
|
|
95
111
|
def call(*args)
|
112
|
+
storage.reset_transaction_depth! if transitions_chain.empty?
|
96
113
|
Result() do
|
97
114
|
after_commit = nil
|
98
115
|
# transaction with support of external calls
|
@@ -110,7 +127,7 @@ module DirtyPipeline
|
|
110
127
|
end
|
111
128
|
|
112
129
|
if fail_cause
|
113
|
-
|
130
|
+
Failure(fail_cause)
|
114
131
|
else
|
115
132
|
Success(destination, output)
|
116
133
|
end
|
@@ -120,9 +137,25 @@ module DirtyPipeline
|
|
120
137
|
end
|
121
138
|
end
|
122
139
|
|
123
|
-
|
140
|
+
def schedule_retry
|
141
|
+
::DirtyPipeline::Worker.perform_in(
|
142
|
+
retry_delay,
|
143
|
+
"enqueued_pipeline" => self.class.to_s,
|
144
|
+
"find_subject_args" => find_subject_args,
|
145
|
+
"retry" => true,
|
146
|
+
)
|
147
|
+
end
|
148
|
+
|
149
|
+
def schedule_cleanup
|
150
|
+
::DirtyPipeline::Worker.perform_in(
|
151
|
+
cleanup_delay,
|
152
|
+
"enqueued_pipeline" => self.class.to_s,
|
153
|
+
"find_subject_args" => find_subject_args,
|
154
|
+
"transition_args" => [Locker::CLEAN],
|
155
|
+
)
|
156
|
+
end
|
124
157
|
|
125
|
-
|
158
|
+
private
|
126
159
|
|
127
160
|
def find_subject_args
|
128
161
|
subject.id
|
@@ -136,29 +169,19 @@ module DirtyPipeline
|
|
136
169
|
self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
|
137
170
|
end
|
138
171
|
|
139
|
-
def
|
140
|
-
|
172
|
+
def transaction(*args)
|
173
|
+
::DirtyPipeline::Transaction.new(self).call(*args) do |*targs|
|
174
|
+
yield(*targs)
|
175
|
+
end
|
141
176
|
end
|
142
177
|
|
143
|
-
def
|
144
|
-
|
145
|
-
Shipping::PipelineWorker.perform_in(
|
146
|
-
retry_delay,
|
147
|
-
"enqueued_pipeline" => self.class.to_s,
|
148
|
-
"find_subject_args" => find_subject_args,
|
149
|
-
"retry" => true,
|
150
|
-
)
|
178
|
+
def Result()
|
179
|
+
status.wrap { yield }
|
151
180
|
end
|
152
181
|
|
153
|
-
def
|
154
|
-
status.error = cause
|
182
|
+
def Failure(cause)
|
155
183
|
storage.fail_event!
|
156
|
-
status.
|
157
|
-
end
|
158
|
-
|
159
|
-
def Exception(error)
|
160
|
-
storage.save_exception!(error)
|
161
|
-
status.error = error
|
184
|
+
status.error = cause
|
162
185
|
status.succeeded = false
|
163
186
|
end
|
164
187
|
|
@@ -172,61 +195,5 @@ module DirtyPipeline
|
|
172
195
|
storage.complete!(output, destination)
|
173
196
|
status.succeeded = true
|
174
197
|
end
|
175
|
-
|
176
|
-
def try_again?(max_attempts_count)
|
177
|
-
return unless max_attempts_count
|
178
|
-
storage.last_event["attempts_count"].to_i < max_attempts_count
|
179
|
-
end
|
180
|
-
|
181
|
-
def find_transition(name)
|
182
|
-
if (const_name = self.class.const_get(name) rescue nil)
|
183
|
-
name = const_name.to_s
|
184
|
-
end
|
185
|
-
self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
|
186
|
-
next if from == Array(storage.status)
|
187
|
-
next if from.include?(storage.status.to_s)
|
188
|
-
raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
|
189
|
-
end
|
190
|
-
end
|
191
|
-
|
192
|
-
def schedule_cleanup
|
193
|
-
Shipping::PipelineWorker.perform_in(
|
194
|
-
cleanup_delay,
|
195
|
-
"enqueued_pipeline" => self.class.to_s,
|
196
|
-
"find_subject_args" => find_subject_args,
|
197
|
-
"transition_args" => [Locker::CLEAN],
|
198
|
-
)
|
199
|
-
end
|
200
|
-
|
201
|
-
def transaction(*args)
|
202
|
-
locker.with_lock(*args) do |transition, *transition_args|
|
203
|
-
begin
|
204
|
-
schedule_cleanup
|
205
|
-
destination, action, max_attempts_count =
|
206
|
-
find_transition(transition).values_at(:to, :action, :attempts)
|
207
|
-
|
208
|
-
status.action_pool.unshift(action)
|
209
|
-
subject.transaction(requires_new: true) do
|
210
|
-
raise ActiveRecord::Rollback if catch(:abort_transaction) do
|
211
|
-
yield(destination, action, *transition_args); nil
|
212
|
-
end
|
213
|
-
end
|
214
|
-
rescue => error
|
215
|
-
if try_again?(max_attempts_count)
|
216
|
-
Retry(error)
|
217
|
-
else
|
218
|
-
Exception(error)
|
219
|
-
end
|
220
|
-
raise
|
221
|
-
ensure
|
222
|
-
if status.succeeded == false
|
223
|
-
status.action_pool.each do |reversable_action|
|
224
|
-
next unless reversable_action.respond_to?(:undo)
|
225
|
-
reversable_action.undo(self, *transition_args)
|
226
|
-
end
|
227
|
-
end
|
228
|
-
end
|
229
|
-
end
|
230
|
-
end
|
231
198
|
end
|
232
199
|
end
|
@@ -16,26 +16,18 @@ module DirtyPipeline
|
|
16
16
|
end
|
17
17
|
|
18
18
|
def success?
|
19
|
-
succeeded
|
19
|
+
!!succeeded
|
20
20
|
end
|
21
21
|
|
22
22
|
def when_success(callback = nil)
|
23
23
|
return self unless success?
|
24
|
-
|
25
|
-
yield(self)
|
26
|
-
else
|
27
|
-
callback.call(self)
|
28
|
-
end
|
24
|
+
block_given? ? yield(self) : callback.(self)
|
29
25
|
self
|
30
26
|
end
|
31
27
|
|
32
28
|
def when_failed(callback = nil)
|
33
29
|
return self unless storage.failed?
|
34
|
-
|
35
|
-
yield(self)
|
36
|
-
else
|
37
|
-
callback.call(self)
|
38
|
-
end
|
30
|
+
block_given? ? yield(self) : callback.(self)
|
39
31
|
self
|
40
32
|
end
|
41
33
|
|
@@ -46,11 +38,7 @@ module DirtyPipeline
|
|
46
38
|
|
47
39
|
def when_error(callback = nil)
|
48
40
|
return self unless errored?
|
49
|
-
|
50
|
-
yield(self)
|
51
|
-
else
|
52
|
-
callback.call(self)
|
53
|
-
end
|
41
|
+
block_given? ? yield(self) : callback.(self)
|
54
42
|
self
|
55
43
|
end
|
56
44
|
|
@@ -60,11 +48,7 @@ module DirtyPipeline
|
|
60
48
|
|
61
49
|
def when_processing(callback = nil)
|
62
50
|
return self unless storage.processing?
|
63
|
-
|
64
|
-
yield(self)
|
65
|
-
else
|
66
|
-
callback.call(self)
|
67
|
-
end
|
51
|
+
block_given? ? yield(self) : callback.(self)
|
68
52
|
self
|
69
53
|
end
|
70
54
|
end
|
@@ -17,19 +17,26 @@ module DirtyPipeline
|
|
17
17
|
def init_store(store_field)
|
18
18
|
self.store = subject.send(store_field).to_h
|
19
19
|
clear! if store.empty?
|
20
|
-
return if
|
20
|
+
return if valid_store?
|
21
21
|
raise InvalidPipelineStorage, store
|
22
22
|
end
|
23
23
|
|
24
|
+
def valid_store?
|
25
|
+
(
|
26
|
+
store.keys &
|
27
|
+
%w(status pipeline_status events errors state transaction_depth)
|
28
|
+
).size == 6
|
29
|
+
end
|
30
|
+
|
24
31
|
def clear
|
25
32
|
self.store = subject.send(
|
26
33
|
"#{field}=",
|
27
34
|
"status" => nil,
|
28
35
|
"pipeline_status" => nil,
|
29
36
|
"state" => {},
|
30
|
-
"cache" => {},
|
31
37
|
"events" => [],
|
32
38
|
"errors" => [],
|
39
|
+
"transaction_depth" => 1
|
33
40
|
)
|
34
41
|
DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
|
35
42
|
end
|
@@ -43,7 +50,8 @@ module DirtyPipeline
|
|
43
50
|
events << {
|
44
51
|
"transition" => transition,
|
45
52
|
"args" => args,
|
46
|
-
"created_at" => Time.now
|
53
|
+
"created_at" => Time.now,
|
54
|
+
"cache" => {},
|
47
55
|
}
|
48
56
|
increment_attempts_count
|
49
57
|
self.pipeline_status = PROCESSING_STATUS
|
@@ -100,7 +108,7 @@ module DirtyPipeline
|
|
100
108
|
|
101
109
|
def commit_pipeline_status!(value = nil)
|
102
110
|
self.pipeline_status = value
|
103
|
-
|
111
|
+
last_event["cache"].clear
|
104
112
|
commit!
|
105
113
|
end
|
106
114
|
alias :reset_pipeline_status! :commit_pipeline_status!
|
@@ -138,6 +146,28 @@ module DirtyPipeline
|
|
138
146
|
errors.last.to_h
|
139
147
|
end
|
140
148
|
|
149
|
+
def reset_transaction_depth
|
150
|
+
store["transaction_depth"] = 1
|
151
|
+
end
|
152
|
+
|
153
|
+
def reset_transaction_depth!
|
154
|
+
reset_transaction_depth
|
155
|
+
commit!
|
156
|
+
end
|
157
|
+
|
158
|
+
def transaction_depth
|
159
|
+
store["transaction_depth"]
|
160
|
+
end
|
161
|
+
|
162
|
+
def increment_transaction_depth
|
163
|
+
store["transaction_depth"] = store["transaction_depth"].to_i + 1
|
164
|
+
end
|
165
|
+
|
166
|
+
def increment_transaction_depth!
|
167
|
+
increment_transaction_depth
|
168
|
+
commit!
|
169
|
+
end
|
170
|
+
|
141
171
|
def increment_attempts_count
|
142
172
|
last_event.merge!(
|
143
173
|
"attempts_count" => last_event["attempts_count"].to_i + 1
|
@@ -0,0 +1,76 @@
|
|
1
|
+
module DirtyPipeline
|
2
|
+
class Transaction
|
3
|
+
attr_reader :locker, :storage, :subject, :pipeline
|
4
|
+
def initialize(pipeline)
|
5
|
+
@pipeline = pipeline
|
6
|
+
@storage = pipeline.storage
|
7
|
+
@subject = pipeline.subject
|
8
|
+
@locker = Locker.new(@subject, @storage)
|
9
|
+
end
|
10
|
+
|
11
|
+
def call(*args)
|
12
|
+
locker.with_lock(*args) do |transition, *transition_args|
|
13
|
+
pipeline.schedule_cleanup
|
14
|
+
begin
|
15
|
+
destination, action, max_attempts_count =
|
16
|
+
find_transition(transition).values_at(:to, :action, :attempts)
|
17
|
+
|
18
|
+
# status.action_pool.unshift(action)
|
19
|
+
subject.transaction(requires_new: true) do
|
20
|
+
raise ActiveRecord::Rollback if catch(:abort_transaction) do
|
21
|
+
yield(destination, action, *transition_args); nil
|
22
|
+
end
|
23
|
+
end
|
24
|
+
rescue => error
|
25
|
+
if try_again?(max_attempts_count)
|
26
|
+
Retry(error)
|
27
|
+
else
|
28
|
+
Exception(error)
|
29
|
+
end
|
30
|
+
raise
|
31
|
+
ensure
|
32
|
+
unless pipeline.status.success?
|
33
|
+
storage.events
|
34
|
+
.last(storage.transaction_depth)
|
35
|
+
.reverse
|
36
|
+
.each do |params|
|
37
|
+
transition = params["transition"]
|
38
|
+
targs = params["args"]
|
39
|
+
reversable_action = find_transition(transition).fetch(:action)
|
40
|
+
reversable_action.undo(self, *targs)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def Retry(error, *args)
|
50
|
+
storage.save_retry!(error)
|
51
|
+
pipeline.schedule_retry
|
52
|
+
end
|
53
|
+
|
54
|
+
def Exception(error)
|
55
|
+
storage.save_exception!(error)
|
56
|
+
pipeline.status.error = error
|
57
|
+
pipeline.status.succeeded = false
|
58
|
+
end
|
59
|
+
|
60
|
+
def try_again?(max_attempts_count)
|
61
|
+
return false unless max_attempts_count
|
62
|
+
storage.last_event["attempts_count"].to_i < max_attempts_count
|
63
|
+
end
|
64
|
+
|
65
|
+
def find_transition(name)
|
66
|
+
if (const_name = pipeline.class.const_get(name) rescue nil)
|
67
|
+
name = const_name.to_s
|
68
|
+
end
|
69
|
+
pipeline.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
|
70
|
+
next if from == Array(storage.status)
|
71
|
+
next if from.include?(storage.status.to_s)
|
72
|
+
raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module DirtyPipeline
|
2
|
+
class Worker
|
3
|
+
include Sidekiq::Worker
|
4
|
+
|
5
|
+
sidekiq_options queue: "default",
|
6
|
+
retry: 1,
|
7
|
+
dead: true
|
8
|
+
|
9
|
+
# args should contain - "enqueued_pipelines" - Array of Pipeline children
|
10
|
+
# args should contain - some args to find_subject
|
11
|
+
# args should contain - some args to make transition
|
12
|
+
|
13
|
+
def perform(options)
|
14
|
+
# FIXME: get rid of ActiveSupport #constantize
|
15
|
+
pipeline_klass = options.fetch("enqueued_pipeline").constantize
|
16
|
+
subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
|
17
|
+
pipeline_klass.new(subject).call(*options.fetch("transition_args"))
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dirty_pipeline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sergey Dolganov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-08-
|
11
|
+
date: 2018-08-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -75,8 +75,10 @@ files:
|
|
75
75
|
- lib/dirty_pipeline/locker.rb
|
76
76
|
- lib/dirty_pipeline/status.rb
|
77
77
|
- lib/dirty_pipeline/storage.rb
|
78
|
+
- lib/dirty_pipeline/transaction.rb
|
78
79
|
- lib/dirty_pipeline/transition.rb
|
79
80
|
- lib/dirty_pipeline/version.rb
|
81
|
+
- lib/dirty_pipeline/worker.rb
|
80
82
|
homepage: https://github.com/sclinede/dirty_pipeline
|
81
83
|
licenses:
|
82
84
|
- MIT
|