dirty_pipeline 0.3.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 6b8f5e951d183bae44a012daeccb345d84b0ccc0dd26edeb1006cfe67cca27ba
4
- data.tar.gz: 2b166ca245ea06ede75099e5c511749e9b9af04414446948e81db9ad5ce2b485
3
+ metadata.gz: 8c98084e4a9f05e60dbfb593de8661371a475d63d8831e80873eb0721b9b5fa6
4
+ data.tar.gz: 445c7d82b17abac6ab4d9c795e1e715bd11ed3bcb944a42e9897ae9c95ef06b9
5
5
  SHA512:
6
- metadata.gz: 0ca3237ae6a6f282040216add989af7a84201d9d1a09be280d177c83de6ff752d55e7d5b3394c1f1e887677ec3e6280feff90f450df5ae18a550873f49bd80b9
7
- data.tar.gz: 0c45a1200ffa8074fda41c1cf7839444d2f72d74a3bf1d4959f8f2a599ffac6aff0eed0a00f973799e4a99f4ef7e4f0fc983301bd39eba0306386592f651aace
6
+ metadata.gz: d1cec2be049b6a10eba0629d04287a685ffa6f00af52333647bc342dad67367a3de662b29d6db6e491b51dcbadf75f3d6b118fcf2bf3b5bf601c102fe522d051
7
+ data.tar.gz: 90ac27df1e8166cc687f76e299350e0a069b983d72c039acd0a4d33881b2b1f9cc728bc0115f3a9af95a0cc781cfba94265838d55ebb8510a90f64e1a4252e0f
@@ -5,6 +5,8 @@ module DirtyPipeline
5
5
  require_relative "dirty_pipeline/storage.rb"
6
6
  require_relative "dirty_pipeline/transition.rb"
7
7
  require_relative "dirty_pipeline/status.rb"
8
+ require_relative "dirty_pipeline/worker.rb"
9
+ require_relative "dirty_pipeline/transaction.rb"
8
10
  require_relative "dirty_pipeline/base.rb"
9
11
 
10
12
  # This method should yield raw Redis connection
@@ -64,8 +64,16 @@ module DirtyPipeline
64
64
  end
65
65
  end
66
66
 
67
+ attr_reader :subject, :error, :storage, :status, :transitions_chain
68
+ def initialize(subject)
69
+ @subject = subject
70
+ @storage = Storage.new(subject, self.class.pipeline_storage)
71
+ @status = Status.new(self)
72
+ @transitions_chain = []
73
+ end
74
+
67
75
  def enqueue(transition_name, *args)
68
- Shipping::PipelineWorker.perform_async(
76
+ DirtyPipeline::Worker.perform_async(
69
77
  "enqueued_pipeline" => self.class.to_s,
70
78
  "find_subject_args" => find_subject_args,
71
79
  "transition_args" => args.unshift(transition_name),
@@ -81,18 +89,27 @@ module DirtyPipeline
81
89
  end
82
90
 
83
91
  def cache
84
- storage.store["cache"]
92
+ storage.last_event["cache"]
85
93
  end
86
94
 
87
- attr_reader :subject, :error, :storage, :status
88
- def initialize(subject)
89
- @subject = subject
90
- @storage = Storage.new(subject, self.class.pipeline_storage)
91
- @locker = Locker.new(@subject, @storage)
92
- @status = Status.new(self)
95
+ def chain(*args)
96
+ transitions_chain << args
97
+ self
98
+ end
99
+
100
+ def execute
101
+ Result() do
102
+ transitions_chain.each do |targs|
103
+ call(*targs)
104
+ storage.increment_transaction_depth!
105
+ end
106
+ storage.reset_transaction_depth!
107
+ transitions_chain.clear
108
+ end
93
109
  end
94
110
 
95
111
  def call(*args)
112
+ storage.reset_transaction_depth! if transitions_chain.empty?
96
113
  Result() do
97
114
  after_commit = nil
98
115
  # transaction with support of external calls
@@ -110,7 +127,7 @@ module DirtyPipeline
110
127
  end
111
128
 
112
129
  if fail_cause
113
- ExpectedError(fail_cause)
130
+ Failure(fail_cause)
114
131
  else
115
132
  Success(destination, output)
116
133
  end
@@ -120,9 +137,25 @@ module DirtyPipeline
120
137
  end
121
138
  end
122
139
 
123
- private
140
+ def schedule_retry
141
+ ::DirtyPipeline::Worker.perform_in(
142
+ retry_delay,
143
+ "enqueued_pipeline" => self.class.to_s,
144
+ "find_subject_args" => find_subject_args,
145
+ "retry" => true,
146
+ )
147
+ end
148
+
149
+ def schedule_cleanup
150
+ ::DirtyPipeline::Worker.perform_in(
151
+ cleanup_delay,
152
+ "enqueued_pipeline" => self.class.to_s,
153
+ "find_subject_args" => find_subject_args,
154
+ "transition_args" => [Locker::CLEAN],
155
+ )
156
+ end
124
157
 
125
- attr_reader :locker
158
+ private
126
159
 
127
160
  def find_subject_args
128
161
  subject.id
@@ -136,29 +169,19 @@ module DirtyPipeline
136
169
  self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
137
170
  end
138
171
 
139
- def Result()
140
- status.wrap { yield }
172
+ def transaction(*args)
173
+ ::DirtyPipeline::Transaction.new(self).call(*args) do |*targs|
174
+ yield(*targs)
175
+ end
141
176
  end
142
177
 
143
- def Retry(error, *args)
144
- storage.save_retry!(error)
145
- Shipping::PipelineWorker.perform_in(
146
- retry_delay,
147
- "enqueued_pipeline" => self.class.to_s,
148
- "find_subject_args" => find_subject_args,
149
- "retry" => true,
150
- )
178
+ def Result()
179
+ status.wrap { yield }
151
180
  end
152
181
 
153
- def ExpectedError(cause)
154
- status.error = cause
182
+ def Failure(cause)
155
183
  storage.fail_event!
156
- status.succeeded = false
157
- end
158
-
159
- def Exception(error)
160
- storage.save_exception!(error)
161
- status.error = error
184
+ status.error = cause
162
185
  status.succeeded = false
163
186
  end
164
187
 
@@ -172,61 +195,5 @@ module DirtyPipeline
172
195
  storage.complete!(output, destination)
173
196
  status.succeeded = true
174
197
  end
175
-
176
- def try_again?(max_attempts_count)
177
- return unless max_attempts_count
178
- storage.last_event["attempts_count"].to_i < max_attempts_count
179
- end
180
-
181
- def find_transition(name)
182
- if (const_name = self.class.const_get(name) rescue nil)
183
- name = const_name.to_s
184
- end
185
- self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
186
- next if from == Array(storage.status)
187
- next if from.include?(storage.status.to_s)
188
- raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
189
- end
190
- end
191
-
192
- def schedule_cleanup
193
- Shipping::PipelineWorker.perform_in(
194
- cleanup_delay,
195
- "enqueued_pipeline" => self.class.to_s,
196
- "find_subject_args" => find_subject_args,
197
- "transition_args" => [Locker::CLEAN],
198
- )
199
- end
200
-
201
- def transaction(*args)
202
- locker.with_lock(*args) do |transition, *transition_args|
203
- begin
204
- schedule_cleanup
205
- destination, action, max_attempts_count =
206
- find_transition(transition).values_at(:to, :action, :attempts)
207
-
208
- status.action_pool.unshift(action)
209
- subject.transaction(requires_new: true) do
210
- raise ActiveRecord::Rollback if catch(:abort_transaction) do
211
- yield(destination, action, *transition_args); nil
212
- end
213
- end
214
- rescue => error
215
- if try_again?(max_attempts_count)
216
- Retry(error)
217
- else
218
- Exception(error)
219
- end
220
- raise
221
- ensure
222
- if status.succeeded == false
223
- status.action_pool.each do |reversable_action|
224
- next unless reversable_action.respond_to?(:undo)
225
- reversable_action.undo(self, *transition_args)
226
- end
227
- end
228
- end
229
- end
230
- end
231
198
  end
232
199
  end
@@ -16,26 +16,18 @@ module DirtyPipeline
16
16
  end
17
17
 
18
18
  def success?
19
- succeeded
19
+ !!succeeded
20
20
  end
21
21
 
22
22
  def when_success(callback = nil)
23
23
  return self unless success?
24
- if block_given?
25
- yield(self)
26
- else
27
- callback.call(self)
28
- end
24
+ block_given? ? yield(self) : callback.(self)
29
25
  self
30
26
  end
31
27
 
32
28
  def when_failed(callback = nil)
33
29
  return self unless storage.failed?
34
- if block_given?
35
- yield(self)
36
- else
37
- callback.call(self)
38
- end
30
+ block_given? ? yield(self) : callback.(self)
39
31
  self
40
32
  end
41
33
 
@@ -46,11 +38,7 @@ module DirtyPipeline
46
38
 
47
39
  def when_error(callback = nil)
48
40
  return self unless errored?
49
- if block_given?
50
- yield(self)
51
- else
52
- callback.call(self)
53
- end
41
+ block_given? ? yield(self) : callback.(self)
54
42
  self
55
43
  end
56
44
 
@@ -60,11 +48,7 @@ module DirtyPipeline
60
48
 
61
49
  def when_processing(callback = nil)
62
50
  return self unless storage.processing?
63
- if block_given?
64
- yield(self)
65
- else
66
- callback.call(self)
67
- end
51
+ block_given? ? yield(self) : callback.(self)
68
52
  self
69
53
  end
70
54
  end
@@ -17,19 +17,26 @@ module DirtyPipeline
17
17
  def init_store(store_field)
18
18
  self.store = subject.send(store_field).to_h
19
19
  clear! if store.empty?
20
- return if (store.keys & %w(cache status events errors state)).size == 5
20
+ return if valid_store?
21
21
  raise InvalidPipelineStorage, store
22
22
  end
23
23
 
24
+ def valid_store?
25
+ (
26
+ store.keys &
27
+ %w(status pipeline_status events errors state transaction_depth)
28
+ ).size == 6
29
+ end
30
+
24
31
  def clear
25
32
  self.store = subject.send(
26
33
  "#{field}=",
27
34
  "status" => nil,
28
35
  "pipeline_status" => nil,
29
36
  "state" => {},
30
- "cache" => {},
31
37
  "events" => [],
32
38
  "errors" => [],
39
+ "transaction_depth" => 1
33
40
  )
34
41
  DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
35
42
  end
@@ -43,7 +50,8 @@ module DirtyPipeline
43
50
  events << {
44
51
  "transition" => transition,
45
52
  "args" => args,
46
- "created_at" => Time.now
53
+ "created_at" => Time.now,
54
+ "cache" => {},
47
55
  }
48
56
  increment_attempts_count
49
57
  self.pipeline_status = PROCESSING_STATUS
@@ -100,7 +108,7 @@ module DirtyPipeline
100
108
 
101
109
  def commit_pipeline_status!(value = nil)
102
110
  self.pipeline_status = value
103
- store["cache"].clear
111
+ last_event["cache"].clear
104
112
  commit!
105
113
  end
106
114
  alias :reset_pipeline_status! :commit_pipeline_status!
@@ -138,6 +146,28 @@ module DirtyPipeline
138
146
  errors.last.to_h
139
147
  end
140
148
 
149
+ def reset_transaction_depth
150
+ store["transaction_depth"] = 1
151
+ end
152
+
153
+ def reset_transaction_depth!
154
+ reset_transaction_depth
155
+ commit!
156
+ end
157
+
158
+ def transaction_depth
159
+ store["transaction_depth"]
160
+ end
161
+
162
+ def increment_transaction_depth
163
+ store["transaction_depth"] = store["transaction_depth"].to_i + 1
164
+ end
165
+
166
+ def increment_transaction_depth!
167
+ increment_transaction_depth
168
+ commit!
169
+ end
170
+
141
171
  def increment_attempts_count
142
172
  last_event.merge!(
143
173
  "attempts_count" => last_event["attempts_count"].to_i + 1
@@ -0,0 +1,76 @@
1
+ module DirtyPipeline
2
+ class Transaction
3
+ attr_reader :locker, :storage, :subject, :pipeline
4
+ def initialize(pipeline)
5
+ @pipeline = pipeline
6
+ @storage = pipeline.storage
7
+ @subject = pipeline.subject
8
+ @locker = Locker.new(@subject, @storage)
9
+ end
10
+
11
+ def call(*args)
12
+ locker.with_lock(*args) do |transition, *transition_args|
13
+ pipeline.schedule_cleanup
14
+ begin
15
+ destination, action, max_attempts_count =
16
+ find_transition(transition).values_at(:to, :action, :attempts)
17
+
18
+ # status.action_pool.unshift(action)
19
+ subject.transaction(requires_new: true) do
20
+ raise ActiveRecord::Rollback if catch(:abort_transaction) do
21
+ yield(destination, action, *transition_args); nil
22
+ end
23
+ end
24
+ rescue => error
25
+ if try_again?(max_attempts_count)
26
+ Retry(error)
27
+ else
28
+ Exception(error)
29
+ end
30
+ raise
31
+ ensure
32
+ unless pipeline.status.success?
33
+ storage.events
34
+ .last(storage.transaction_depth)
35
+ .reverse
36
+ .each do |params|
37
+ transition = params["transition"]
38
+ targs = params["args"]
39
+ reversable_action = find_transition(transition).fetch(:action)
40
+ reversable_action.undo(self, *targs)
41
+ end
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ private
48
+
49
+ def Retry(error, *args)
50
+ storage.save_retry!(error)
51
+ pipeline.schedule_retry
52
+ end
53
+
54
+ def Exception(error)
55
+ storage.save_exception!(error)
56
+ pipeline.status.error = error
57
+ pipeline.status.succeeded = false
58
+ end
59
+
60
+ def try_again?(max_attempts_count)
61
+ return false unless max_attempts_count
62
+ storage.last_event["attempts_count"].to_i < max_attempts_count
63
+ end
64
+
65
+ def find_transition(name)
66
+ if (const_name = pipeline.class.const_get(name) rescue nil)
67
+ name = const_name.to_s
68
+ end
69
+ pipeline.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
70
+ next if from == Array(storage.status)
71
+ next if from.include?(storage.status.to_s)
72
+ raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
73
+ end
74
+ end
75
+ end
76
+ end
@@ -1,3 +1,3 @@
1
1
  module DirtyPipeline
2
- VERSION = "0.3.0"
2
+ VERSION = "0.4.0"
3
3
  end
@@ -0,0 +1,20 @@
1
+ module DirtyPipeline
2
+ class Worker
3
+ include Sidekiq::Worker
4
+
5
+ sidekiq_options queue: "default",
6
+ retry: 1,
7
+ dead: true
8
+
9
+ # args should contain - "enqueued_pipelines" - Array of Pipeline children
10
+ # args should contain - some args to find_subject
11
+ # args should contain - some args to make transition
12
+
13
+ def perform(options)
14
+ # FIXME: get rid of ActiveSupport #constantize
15
+ pipeline_klass = options.fetch("enqueued_pipeline").constantize
16
+ subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
17
+ pipeline_klass.new(subject).call(*options.fetch("transition_args"))
18
+ end
19
+ end
20
+ end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dirty_pipeline
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.3.0
4
+ version: 0.4.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sergey Dolganov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-08-24 00:00:00.000000000 Z
11
+ date: 2018-08-25 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -75,8 +75,10 @@ files:
75
75
  - lib/dirty_pipeline/locker.rb
76
76
  - lib/dirty_pipeline/status.rb
77
77
  - lib/dirty_pipeline/storage.rb
78
+ - lib/dirty_pipeline/transaction.rb
78
79
  - lib/dirty_pipeline/transition.rb
79
80
  - lib/dirty_pipeline/version.rb
81
+ - lib/dirty_pipeline/worker.rb
80
82
  homepage: https://github.com/sclinede/dirty_pipeline
81
83
  licenses:
82
84
  - MIT