dirty_pipeline 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/dirty_pipeline.rb +6 -3
- data/lib/dirty_pipeline/base.rb +119 -121
- data/lib/dirty_pipeline/event.rb +107 -0
- data/lib/dirty_pipeline/ext/camelcase.rb +63 -0
- data/lib/dirty_pipeline/queue.rb +82 -0
- data/lib/dirty_pipeline/railway.rb +74 -0
- data/lib/dirty_pipeline/status.rb +14 -42
- data/lib/dirty_pipeline/storage.rb +45 -179
- data/lib/dirty_pipeline/transaction.rb +37 -57
- data/lib/dirty_pipeline/transition.rb +22 -13
- data/lib/dirty_pipeline/version.rb +1 -1
- data/lib/dirty_pipeline/worker.rb +17 -10
- metadata +6 -3
- data/lib/dirty_pipeline/locker.rb +0 -108
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2d7bb36236434b64e26818af5a9ba0aa299035304fbbedc2dc305111b93b58a2
|
4
|
+
data.tar.gz: efed49a5b15c73610497df6ca1d6e05018efa586fbd8ae3d11df5f6092414fe6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c4324152fcc2d3e403ae93e23916fa2745820d199ed277cdbeef6d93b5d8dc726235a368de1c6e95fe6b124442c32e0b7e98fd531b74baef29b197c8f2bf702d
|
7
|
+
data.tar.gz: ec4cf7fbf3e02ec46bd89381f7534bde8a023cfa07f5cf0c3bfc5f3b040be49e7e4457ecfec6677fd01e7f2832b3012b7a824cf199acc94db5f6cb323d1fab13
|
data/lib/dirty_pipeline.rb
CHANGED
@@ -1,13 +1,16 @@
|
|
1
1
|
require "dirty_pipeline/version"
|
2
2
|
|
3
3
|
module DirtyPipeline
|
4
|
-
require_relative "dirty_pipeline/
|
5
|
-
require_relative "dirty_pipeline/storage.rb"
|
6
|
-
require_relative "dirty_pipeline/transition.rb"
|
4
|
+
require_relative "dirty_pipeline/ext/camelcase.rb"
|
7
5
|
require_relative "dirty_pipeline/status.rb"
|
6
|
+
require_relative "dirty_pipeline/storage.rb"
|
8
7
|
require_relative "dirty_pipeline/worker.rb"
|
9
8
|
require_relative "dirty_pipeline/transaction.rb"
|
9
|
+
require_relative "dirty_pipeline/event.rb"
|
10
|
+
require_relative "dirty_pipeline/queue.rb"
|
11
|
+
require_relative "dirty_pipeline/railway.rb"
|
10
12
|
require_relative "dirty_pipeline/base.rb"
|
13
|
+
require_relative "dirty_pipeline/transition.rb"
|
11
14
|
|
12
15
|
# This method should yield raw Redis connection
|
13
16
|
def self.with_redis
|
data/lib/dirty_pipeline/base.rb
CHANGED
@@ -2,14 +2,7 @@ module DirtyPipeline
|
|
2
2
|
class Base
|
3
3
|
DEFAULT_RETRY_DELAY = 5 * 60 # 5 minutes
|
4
4
|
DEFAULT_CLEANUP_DELAY = 60 * 60 * 24 # 1 day
|
5
|
-
|
6
|
-
Storage::FAILED_STATUS,
|
7
|
-
Storage::PROCESSING_STATUS,
|
8
|
-
Storage::RETRY_STATUS,
|
9
|
-
Locker::CLEAN,
|
10
|
-
]
|
11
|
-
|
12
|
-
class ReservedStatusError < StandardError; end
|
5
|
+
|
13
6
|
class InvalidTransition < StandardError; end
|
14
7
|
|
15
8
|
class << self
|
@@ -19,144 +12,153 @@ module DirtyPipeline
|
|
19
12
|
|
20
13
|
attr_reader :transitions_map
|
21
14
|
def inherited(child)
|
22
|
-
child.instance_variable_set(
|
15
|
+
child.instance_variable_set(
|
16
|
+
:@transitions_map,
|
17
|
+
transitions_map || Hash.new
|
18
|
+
)
|
23
19
|
end
|
24
|
-
# PG JSONB column
|
25
|
-
# {
|
26
|
-
# status: :errored,
|
27
|
-
# state: {
|
28
|
-
# field: "value",
|
29
|
-
# },
|
30
|
-
# errors: [
|
31
|
-
# {
|
32
|
-
# error: "RuPost::API::Error",
|
33
|
-
# error_message: "Timeout error",
|
34
|
-
# created_at: 2018-01-01T13:22Z
|
35
|
-
# },
|
36
|
-
# ],
|
37
|
-
# events: [
|
38
|
-
# {
|
39
|
-
# action: Init,
|
40
|
-
# input: ...,
|
41
|
-
# created_at: ...,
|
42
|
-
# updated_at: ...,
|
43
|
-
# attempts_count: 2,
|
44
|
-
# },
|
45
|
-
# {...},
|
46
|
-
# ]
|
47
|
-
# }
|
48
20
|
attr_accessor :pipeline_storage, :retry_delay, :cleanup_delay
|
49
21
|
|
50
|
-
|
51
|
-
|
52
|
-
|
22
|
+
using StringCamelcase
|
23
|
+
|
24
|
+
def transition(name, from:, to:, action: nil, attempts: 1)
|
25
|
+
action ||= const_get(name.to_s.camelcase)
|
26
|
+
@transitions_map[name.to_s] = {
|
53
27
|
action: action,
|
54
28
|
from: Array(from).map(&:to_s),
|
55
29
|
to: to.to_s,
|
56
30
|
attempts: attempts,
|
57
31
|
}
|
58
32
|
end
|
59
|
-
|
60
|
-
private
|
61
|
-
|
62
|
-
def valid_statuses?(from, to)
|
63
|
-
((Array(to) + Array(from)) & RESERVED_STATUSES).empty?
|
64
|
-
end
|
65
33
|
end
|
66
34
|
|
67
|
-
attr_reader :subject, :
|
68
|
-
def initialize(subject)
|
35
|
+
attr_reader :subject, :storage, :status, :uuid, :queue, :railway
|
36
|
+
def initialize(subject, uuid: nil)
|
37
|
+
@uuid = uuid || Nanoid.generate
|
69
38
|
@subject = subject
|
70
39
|
@storage = Storage.new(subject, self.class.pipeline_storage)
|
71
|
-
@
|
72
|
-
@
|
40
|
+
@railway = Railway.new(subject, @uuid)
|
41
|
+
@status = Status.success(subject)
|
73
42
|
end
|
74
43
|
|
75
|
-
def
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
44
|
+
def find_transition(name)
|
45
|
+
self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
|
46
|
+
next unless railway.operation.eql?(:call)
|
47
|
+
next if from == Array(storage.status)
|
48
|
+
next if from.include?(storage.status.to_s)
|
49
|
+
raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
|
50
|
+
end
|
81
51
|
end
|
82
52
|
|
83
53
|
def reset!
|
84
|
-
|
54
|
+
railway.clear!
|
85
55
|
end
|
86
56
|
|
87
57
|
def clear!
|
88
58
|
storage.clear!
|
89
|
-
|
90
|
-
|
91
|
-
def cache
|
92
|
-
storage.last_event["cache"]
|
59
|
+
reset!
|
93
60
|
end
|
94
61
|
|
95
62
|
def chain(*args)
|
96
|
-
|
63
|
+
railway[:call] << Event.create(*args, tx_id: @uuid)
|
97
64
|
self
|
98
65
|
end
|
99
66
|
|
100
|
-
def
|
101
|
-
|
102
|
-
|
103
|
-
call(*targs)
|
104
|
-
storage.increment_transaction_depth!
|
105
|
-
end
|
106
|
-
storage.reset_transaction_depth!
|
107
|
-
transitions_chain.clear
|
108
|
-
end
|
67
|
+
def call
|
68
|
+
return self if (serialized_event = railway.next).nil?
|
69
|
+
execute(load_event(serialized_event))
|
109
70
|
end
|
71
|
+
alias :call_next :call
|
110
72
|
|
111
|
-
def
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
transaction(*args) do |destination, action, *targs|
|
117
|
-
output = {}
|
118
|
-
fail_cause = nil
|
119
|
-
|
120
|
-
output, *after_commit = catch(:success) do
|
121
|
-
fail_cause = catch(:fail_with_error) do
|
122
|
-
Abort() if catch(:abort) do
|
123
|
-
throw :success, action.(self, *targs)
|
124
|
-
end
|
125
|
-
end
|
126
|
-
nil
|
127
|
-
end
|
73
|
+
def clean
|
74
|
+
railway.switch_to(:undo)
|
75
|
+
call_next
|
76
|
+
self
|
77
|
+
end
|
128
78
|
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
end
|
134
|
-
end
|
79
|
+
def retry
|
80
|
+
return unless (event = load_event(railway.queue.processing_event))
|
81
|
+
execute(event, :retry)
|
82
|
+
end
|
135
83
|
|
136
|
-
|
137
|
-
|
84
|
+
def schedule_cleanup
|
85
|
+
schedule("cleanup", cleanup_delay)
|
138
86
|
end
|
139
87
|
|
140
88
|
def schedule_retry
|
141
|
-
|
142
|
-
retry_delay,
|
143
|
-
"enqueued_pipeline" => self.class.to_s,
|
144
|
-
"find_subject_args" => find_subject_args,
|
145
|
-
"retry" => true,
|
146
|
-
)
|
89
|
+
schedule("retry", retry_delay)
|
147
90
|
end
|
148
91
|
|
149
|
-
def
|
150
|
-
|
151
|
-
|
92
|
+
def schedule(operation, delay = nil)
|
93
|
+
job_args = {
|
94
|
+
"transaction_id" => @uuid,
|
152
95
|
"enqueued_pipeline" => self.class.to_s,
|
153
96
|
"find_subject_args" => find_subject_args,
|
154
|
-
"
|
155
|
-
|
97
|
+
"operation" => operation,
|
98
|
+
}
|
99
|
+
|
100
|
+
if delay.nil?
|
101
|
+
::DirtyPipeline::Worker.perform_async(job_args)
|
102
|
+
else
|
103
|
+
::DirtyPipeline::Worker.perform_in(delay, job_args)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def when_success
|
108
|
+
yield(status.data, self) if status.success?
|
109
|
+
self
|
110
|
+
end
|
111
|
+
|
112
|
+
def when_failure(tag = status.tag)
|
113
|
+
yield(status.data, self) if status.failure? && status.tag == tag
|
114
|
+
self
|
156
115
|
end
|
157
116
|
|
158
117
|
private
|
159
118
|
|
119
|
+
def execute(event, type = :call)
|
120
|
+
transaction(event).public_send(type) do |destination, action, *args|
|
121
|
+
state_changes = process_action(action, event, *args)
|
122
|
+
next if status.failure?
|
123
|
+
Success(event, state_changes, destination)
|
124
|
+
end
|
125
|
+
call_next
|
126
|
+
|
127
|
+
self
|
128
|
+
end
|
129
|
+
|
130
|
+
def load_event(enqueued_event)
|
131
|
+
storage.find_event(enqueued_event.id) || enqueued_event
|
132
|
+
end
|
133
|
+
|
134
|
+
def process_action(action, event, *args)
|
135
|
+
return catch(:success) do
|
136
|
+
return if interupt_on_error(event) do
|
137
|
+
return if interupt_on_abort(event) do
|
138
|
+
throw :success, run_operation(action, event, *args); nil
|
139
|
+
end
|
140
|
+
end
|
141
|
+
nil
|
142
|
+
end
|
143
|
+
rescue => exception
|
144
|
+
@status = Status.failure(exception, tag: :exception)
|
145
|
+
raise
|
146
|
+
end
|
147
|
+
|
148
|
+
def run_operation(action, event, *args)
|
149
|
+
return unless action.respond_to?(operation = railway.operation)
|
150
|
+
action.public_send(operation, event, self, *args)
|
151
|
+
end
|
152
|
+
|
153
|
+
def interupt_on_error(event)
|
154
|
+
return if (fail_cause = catch(:fail_with_error) { yield; nil }).nil?
|
155
|
+
Failure(event, fail_cause)
|
156
|
+
end
|
157
|
+
|
158
|
+
def interupt_on_abort(event)
|
159
|
+
Abort(event) if catch(:abort) { yield; nil }
|
160
|
+
end
|
161
|
+
|
160
162
|
def find_subject_args
|
161
163
|
subject.id
|
162
164
|
end
|
@@ -169,31 +171,27 @@ module DirtyPipeline
|
|
169
171
|
self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
|
170
172
|
end
|
171
173
|
|
172
|
-
def transaction(
|
173
|
-
::DirtyPipeline::Transaction.new(self
|
174
|
-
yield(*targs)
|
175
|
-
end
|
176
|
-
end
|
177
|
-
|
178
|
-
def Result()
|
179
|
-
status.wrap { yield }
|
174
|
+
def transaction(event)
|
175
|
+
::DirtyPipeline::Transaction.new(self, railway.queue, event)
|
180
176
|
end
|
181
177
|
|
182
|
-
def Failure(cause)
|
183
|
-
|
184
|
-
|
185
|
-
status
|
178
|
+
def Failure(event, cause)
|
179
|
+
event.failure!
|
180
|
+
railway.switch_to(:undo)
|
181
|
+
@status = Status.failure(cause, tag: :error)
|
182
|
+
throw :abort_transaction, true
|
186
183
|
end
|
187
184
|
|
188
|
-
def Abort()
|
189
|
-
|
185
|
+
def Abort(event)
|
186
|
+
event.failure!
|
187
|
+
railway.switch_to(:undo)
|
188
|
+
@status = Status.failure(subject, tag: :aborted)
|
190
189
|
throw :abort_transaction, true
|
191
190
|
end
|
192
191
|
|
193
|
-
def Success(
|
194
|
-
|
195
|
-
|
196
|
-
status.succeeded = true
|
192
|
+
def Success(event, changes, destination)
|
193
|
+
event.complete(changes, destination)
|
194
|
+
@status = Status.success(subject)
|
197
195
|
end
|
198
196
|
end
|
199
197
|
end
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module DirtyPipeline
|
4
|
+
class Event
|
5
|
+
NEW = "new".freeze
|
6
|
+
START = "started".freeze
|
7
|
+
FAILURE = "failed".freeze
|
8
|
+
RETRY = "retry".freeze
|
9
|
+
SUCCESS = "success".freeze
|
10
|
+
|
11
|
+
def self.create(transition, *args, tx_id:)
|
12
|
+
new(
|
13
|
+
data: {
|
14
|
+
"uuid" => Nanoid.generate,
|
15
|
+
"transaction_uuid" => tx_id,
|
16
|
+
"transition" => transition,
|
17
|
+
"args" => args,
|
18
|
+
}
|
19
|
+
)
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.load(json)
|
23
|
+
return unless json
|
24
|
+
new(JSON.load(json))
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.dump(event)
|
28
|
+
JSON.dump(event.to_h)
|
29
|
+
end
|
30
|
+
|
31
|
+
def dump
|
32
|
+
self.class.dump(self)
|
33
|
+
end
|
34
|
+
|
35
|
+
attr_reader :id, :tx_id, :error, :data
|
36
|
+
def initialize(options = {}, data: nil, error: nil)
|
37
|
+
unless options.empty?
|
38
|
+
options_hash = options.to_h
|
39
|
+
data ||= options_hash["data"]
|
40
|
+
error ||= options_hash["error"]
|
41
|
+
transition = options_hash["transition"]
|
42
|
+
args = options_hash["args"]
|
43
|
+
end
|
44
|
+
|
45
|
+
data_hash = data.to_h
|
46
|
+
|
47
|
+
@tx_id = data_hash.fetch("transaction_uuid")
|
48
|
+
@id = data_hash.fetch("uuid")
|
49
|
+
@data = {
|
50
|
+
"uuid" => @id,
|
51
|
+
"transaction_uuid" => @tx_id,
|
52
|
+
"transition" => transition,
|
53
|
+
"args" => args,
|
54
|
+
"created_at" => Time.now,
|
55
|
+
"cache" => {},
|
56
|
+
"attempts_count" => 1,
|
57
|
+
"status" => NEW,
|
58
|
+
}.merge(data_hash)
|
59
|
+
@error = error
|
60
|
+
end
|
61
|
+
|
62
|
+
def to_h
|
63
|
+
{data: @data, error: @error}
|
64
|
+
end
|
65
|
+
|
66
|
+
%w(args transition cache destination changes).each do |method_name|
|
67
|
+
define_method("#{method_name}") { @data[method_name] }
|
68
|
+
end
|
69
|
+
|
70
|
+
%w(new start retry failure).each do |method_name|
|
71
|
+
define_method("#{method_name}?") do
|
72
|
+
@data["status"] == self.class.const_get(method_name.upcase)
|
73
|
+
end
|
74
|
+
|
75
|
+
define_method("#{method_name}!") do
|
76
|
+
@data["status"] = self.class.const_get(method_name.upcase)
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
def link_exception(exception)
|
81
|
+
@error = {
|
82
|
+
"exception" => exception.class.to_s,
|
83
|
+
"exception_message" => exception.message,
|
84
|
+
"created_at" => Time.current,
|
85
|
+
}
|
86
|
+
failure!
|
87
|
+
end
|
88
|
+
|
89
|
+
def attempts_count
|
90
|
+
@data["attempts_count"].to_i
|
91
|
+
end
|
92
|
+
|
93
|
+
def attempt_retry
|
94
|
+
@data["updated_at"] = Time.now
|
95
|
+
@data["attempts_count"] = attempts_count + 1
|
96
|
+
end
|
97
|
+
|
98
|
+
def complete(changes, destination)
|
99
|
+
@data.merge!(
|
100
|
+
"destination" => destination,
|
101
|
+
"changes" => changes,
|
102
|
+
"updated_at" => Time.now,
|
103
|
+
"status" => SUCCESS,
|
104
|
+
)
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -0,0 +1,63 @@
|
|
1
|
+
module DirtyPipeline
|
2
|
+
module StringCamelcase
|
3
|
+
refine String do
|
4
|
+
# rubocop:disable Metrics/LineLength
|
5
|
+
# See gem Facets String#camelcase
|
6
|
+
#
|
7
|
+
# Converts a string to camelcase. This method leaves the first character
|
8
|
+
# as given. This allows other methods to be used first, such as #uppercase
|
9
|
+
# and #lowercase.
|
10
|
+
#
|
11
|
+
# "camel_case".camelcase #=> "camelCase"
|
12
|
+
# "Camel_case".camelcase #=> "CamelCase"
|
13
|
+
#
|
14
|
+
# Custom +separators+ can be used to specify the patterns used to determine
|
15
|
+
# where capitalization should occur. By default these are underscores (`_`)
|
16
|
+
# and space characters (`\s`).
|
17
|
+
#
|
18
|
+
# "camel/case".camelcase('/') #=> "camelCase"
|
19
|
+
#
|
20
|
+
# If the first separator is a symbol, either `:lower` or `:upper`, then
|
21
|
+
# the first characters of the string will be downcased or upcased respectively.
|
22
|
+
#
|
23
|
+
# "camel_case".camelcase(:upper) #=> "CamelCase"
|
24
|
+
#
|
25
|
+
# Note that this implementation is different from ActiveSupport's.
|
26
|
+
# If that is what you are looking for you may want {#modulize}.
|
27
|
+
#
|
28
|
+
# rubocop:enable Metrics/LineLength
|
29
|
+
|
30
|
+
# rubocop:disable Metrics/AbcSize,Metrics/MethodLength
|
31
|
+
def camelcase(*separators)
|
32
|
+
case separators.first
|
33
|
+
when Symbol, TrueClass, FalseClass, NilClass
|
34
|
+
first_letter = separators.shift
|
35
|
+
end
|
36
|
+
|
37
|
+
separators = ["_", '\s'] if separators.empty?
|
38
|
+
|
39
|
+
str = dup
|
40
|
+
|
41
|
+
separators.each do |s|
|
42
|
+
str = str.gsub(/(?:#{s}+)([a-z])/) do
|
43
|
+
Regexp.last_match(1).upcase
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
case first_letter
|
48
|
+
when :upper, true
|
49
|
+
str = str.gsub(/(\A|\s)([a-z])/) do
|
50
|
+
Regexp.last_match(1) + Regexp.last_match(2).upcase
|
51
|
+
end
|
52
|
+
when :lower, false
|
53
|
+
str = str.gsub(/(\A|\s)([A-Z])/) do
|
54
|
+
Regexp.last_match(1) + Regexp.last_match(2).downcase
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
str
|
59
|
+
end
|
60
|
+
# rubocop:enable Metrics/AbcSize,Metrics/MethodLength
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,82 @@
|
|
1
|
+
module DirtyPipeline
|
2
|
+
class Queue
|
3
|
+
attr_reader :root
|
4
|
+
def initialize(operation, subject, transaction_id)
|
5
|
+
@root = "dirty-pipeline-queue:#{subject.class}:#{subject.id}:" \
|
6
|
+
"op_#{operation}:txid_#{transaction_id}"
|
7
|
+
end
|
8
|
+
|
9
|
+
def clear!
|
10
|
+
DirtyPipeline.with_redis do |r|
|
11
|
+
r.del active_event_key
|
12
|
+
r.del events_queue_key
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def to_a
|
17
|
+
DirtyPipeline.with_redis { |r| r.lrange(events_queue_key, 0, -1) }
|
18
|
+
end
|
19
|
+
|
20
|
+
def push(event)
|
21
|
+
DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
|
22
|
+
end
|
23
|
+
alias :<< :push
|
24
|
+
|
25
|
+
def unshift(event)
|
26
|
+
DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
|
27
|
+
end
|
28
|
+
|
29
|
+
def dequeue
|
30
|
+
DirtyPipeline.with_redis do |r|
|
31
|
+
data = r.lpop(events_queue_key)
|
32
|
+
data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
|
33
|
+
return unpack(data)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
alias :pop :dequeue
|
37
|
+
|
38
|
+
def event_in_progress?(event = nil)
|
39
|
+
if event.nil?
|
40
|
+
!processing_event.nil?
|
41
|
+
else
|
42
|
+
processing_event.id == event.id
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def processing_event
|
47
|
+
DirtyPipeline.with_redis { |r| unpack r.get(active_event_key) }
|
48
|
+
end
|
49
|
+
|
50
|
+
private
|
51
|
+
|
52
|
+
def pack(event)
|
53
|
+
JSON.dump(
|
54
|
+
"evid" => event.id,
|
55
|
+
"txid" => event.tx_id,
|
56
|
+
"transit" => event.transition,
|
57
|
+
"args" => event.args,
|
58
|
+
)
|
59
|
+
end
|
60
|
+
|
61
|
+
def unpack(packed_event)
|
62
|
+
return unless packed_event
|
63
|
+
unpacked_event = JSON.load(packed_event)
|
64
|
+
Event.new(
|
65
|
+
data: {
|
66
|
+
"uuid" => unpacked_event["evid"],
|
67
|
+
"transaction_uuid" => unpacked_event["txid"],
|
68
|
+
"transition" => unpacked_event["transit"],
|
69
|
+
"args" => unpacked_event["args"],
|
70
|
+
}
|
71
|
+
)
|
72
|
+
end
|
73
|
+
|
74
|
+
def events_queue_key
|
75
|
+
"#{root}:events"
|
76
|
+
end
|
77
|
+
|
78
|
+
def active_event_key
|
79
|
+
"#{root}:active"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
@@ -0,0 +1,74 @@
|
|
1
|
+
module DirtyPipeline
|
2
|
+
class Railway
|
3
|
+
OPERATIONS = %w(call undo finalize)
|
4
|
+
|
5
|
+
def initialize(subject, transaction_id)
|
6
|
+
@tx_id = transaction_id
|
7
|
+
@root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:" \
|
8
|
+
":txid_#{transaction_id}"
|
9
|
+
@queues = Hash[
|
10
|
+
OPERATIONS.map do |operation|
|
11
|
+
[operation, Queue.new(operation, subject, transaction_id)]
|
12
|
+
end
|
13
|
+
]
|
14
|
+
end
|
15
|
+
|
16
|
+
def clear!
|
17
|
+
@queues.values.each(&:clear!)
|
18
|
+
DirtyPipeline.with_redis { |r| r.del(active_operation_key) }
|
19
|
+
end
|
20
|
+
|
21
|
+
def next
|
22
|
+
return if other_transaction_in_progress?
|
23
|
+
start_transaction! if running_transaction.nil?
|
24
|
+
|
25
|
+
queue.pop.tap { |event| finish_transaction! if event.nil? }
|
26
|
+
end
|
27
|
+
|
28
|
+
def queue(name = active)
|
29
|
+
@queues[name.to_s]
|
30
|
+
end
|
31
|
+
alias :[] :queue
|
32
|
+
|
33
|
+
def switch_to(name)
|
34
|
+
raise ArgumentError unless OPERATIONS.include?(name.to_s)
|
35
|
+
return if name.to_s == active
|
36
|
+
DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
|
37
|
+
end
|
38
|
+
|
39
|
+
def active
|
40
|
+
DirtyPipeline.with_redis { |r| r.get(active_operation_key) }
|
41
|
+
end
|
42
|
+
alias :operation :active
|
43
|
+
|
44
|
+
private
|
45
|
+
|
46
|
+
def active_transaction_key
|
47
|
+
"#{@root}:active_transaction"
|
48
|
+
end
|
49
|
+
|
50
|
+
def active_operation_key
|
51
|
+
"#{@root}:active_operation"
|
52
|
+
end
|
53
|
+
|
54
|
+
def start_transaction!
|
55
|
+
switch_to(OPERATIONS.first)
|
56
|
+
DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
|
57
|
+
end
|
58
|
+
|
59
|
+
def finish_transaction!
|
60
|
+
return unless running_transaction == @tx_id
|
61
|
+
DirtyPipeline.with_redis { |r| r.del(active_transaction_key) }
|
62
|
+
@queues.values.each(&:clear!)
|
63
|
+
end
|
64
|
+
|
65
|
+
def running_transaction
|
66
|
+
DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
|
67
|
+
end
|
68
|
+
|
69
|
+
def other_transaction_in_progress?
|
70
|
+
return false if running_transaction.nil?
|
71
|
+
running_transaction != @tx_id
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
@@ -1,55 +1,27 @@
|
|
1
1
|
module DirtyPipeline
|
2
|
-
class Status
|
3
|
-
|
4
|
-
attr_reader :storage, :pipeline
|
5
|
-
def initialize(*)
|
6
|
-
super
|
7
|
-
@storage = __getobj__.storage
|
8
|
-
@action_pool = []
|
9
|
-
end
|
2
|
+
class Status
|
3
|
+
attr_reader :success, :tag, :data
|
10
4
|
|
11
|
-
def
|
12
|
-
|
13
|
-
self.succeeded = nil
|
14
|
-
yield
|
15
|
-
self
|
5
|
+
def self.success(data, tag: :success)
|
6
|
+
new(true, data, tag)
|
16
7
|
end
|
17
8
|
|
18
|
-
def
|
19
|
-
|
9
|
+
def self.failure(data, tag: :exception)
|
10
|
+
new(false, data, tag)
|
20
11
|
end
|
21
12
|
|
22
|
-
def
|
23
|
-
|
24
|
-
|
25
|
-
|
13
|
+
def initialize(success, data, tag = nil)
|
14
|
+
@success = success
|
15
|
+
@data = data
|
16
|
+
@tag = tag
|
26
17
|
end
|
27
18
|
|
28
|
-
def
|
29
|
-
|
30
|
-
block_given? ? yield(self) : callback.(self)
|
31
|
-
self
|
32
|
-
end
|
33
|
-
|
34
|
-
def errored?
|
35
|
-
return if succeeded.nil?
|
36
|
-
ready? && !succeeded
|
37
|
-
end
|
38
|
-
|
39
|
-
def when_error(callback = nil)
|
40
|
-
return self unless errored?
|
41
|
-
block_given? ? yield(self) : callback.(self)
|
42
|
-
self
|
43
|
-
end
|
44
|
-
|
45
|
-
def ready?
|
46
|
-
storage.pipeline_status.nil?
|
19
|
+
def success?
|
20
|
+
!!success
|
47
21
|
end
|
48
22
|
|
49
|
-
def
|
50
|
-
|
51
|
-
block_given? ? yield(self) : callback.(self)
|
52
|
-
self
|
23
|
+
def failure?
|
24
|
+
!success?
|
53
25
|
end
|
54
26
|
end
|
55
27
|
end
|
@@ -1,11 +1,12 @@
|
|
1
1
|
module DirtyPipeline
|
2
2
|
class Storage
|
3
|
-
|
3
|
+
SUCCESS_STATUS = "success".freeze
|
4
|
+
FAILURE_STATUS = "failure".freeze
|
4
5
|
RETRY_STATUS = "retry".freeze
|
5
6
|
PROCESSING_STATUS = "processing".freeze
|
6
7
|
class InvalidPipelineStorage < StandardError; end
|
7
8
|
|
8
|
-
attr_reader :subject, :field
|
9
|
+
attr_reader :subject, :field, :transactions_queue
|
9
10
|
attr_accessor :store
|
10
11
|
alias :to_h :store
|
11
12
|
def initialize(subject, field)
|
@@ -16,210 +17,75 @@ module DirtyPipeline
|
|
16
17
|
|
17
18
|
def init_store(store_field)
|
18
19
|
self.store = subject.send(store_field).to_h
|
19
|
-
clear
|
20
|
+
clear if store.empty?
|
20
21
|
return if valid_store?
|
21
22
|
raise InvalidPipelineStorage, store
|
22
23
|
end
|
23
24
|
|
24
25
|
def valid_store?
|
25
|
-
(
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
26
|
+
(store.keys & %w(status events errors state)).size.eql?(4)
|
27
|
+
end
|
28
|
+
|
29
|
+
# PG JSONB column
|
30
|
+
# {
|
31
|
+
# status: :errored,
|
32
|
+
# state: {
|
33
|
+
# field: "value",
|
34
|
+
# },
|
35
|
+
# errors: {
|
36
|
+
# "<event_id>": {
|
37
|
+
# error: "RuPost::API::Error",
|
38
|
+
# error_message: "Timeout error",
|
39
|
+
# created_at: 2018-01-01T13:22Z
|
40
|
+
# },
|
41
|
+
# },
|
42
|
+
# events: {
|
43
|
+
# <event_id>: {
|
44
|
+
# action: Init,
|
45
|
+
# input: ...,
|
46
|
+
# created_at: ...,
|
47
|
+
# updated_at: ...,
|
48
|
+
# attempts_count: 2,
|
49
|
+
# },
|
50
|
+
# <event_id>: {...},
|
51
|
+
# }
|
52
|
+
# }
|
31
53
|
def clear
|
32
54
|
self.store = subject.send(
|
33
55
|
"#{field}=",
|
34
56
|
"status" => nil,
|
35
|
-
"pipeline_status" => nil,
|
36
57
|
"state" => {},
|
37
|
-
"events" =>
|
38
|
-
"errors" =>
|
39
|
-
"transaction_depth" => 1
|
58
|
+
"events" => {},
|
59
|
+
"errors" => {}
|
40
60
|
)
|
41
|
-
DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
|
42
61
|
end
|
43
62
|
|
44
63
|
def clear!
|
45
64
|
clear
|
46
|
-
|
47
|
-
end
|
48
|
-
|
49
|
-
def start!(transition, args)
|
50
|
-
events << {
|
51
|
-
"transition" => transition,
|
52
|
-
"args" => args,
|
53
|
-
"created_at" => Time.now,
|
54
|
-
"cache" => {},
|
55
|
-
}
|
56
|
-
increment_attempts_count
|
57
|
-
self.pipeline_status = PROCESSING_STATUS
|
58
|
-
# self.status = "processing", should be set by Locker
|
59
|
-
commit!
|
60
|
-
end
|
61
|
-
|
62
|
-
def start_retry!
|
63
|
-
last_event.merge!(updated_at: Time.now)
|
64
|
-
increment_attempts_count
|
65
|
-
self.pipeline_status = PROCESSING_STATUS
|
66
|
-
# self.status = "processing", should be set by Locker
|
67
|
-
commit!
|
68
|
-
end
|
69
|
-
|
70
|
-
def complete!(output, destination)
|
71
|
-
store["status"] = destination
|
72
|
-
state.merge!(output)
|
73
|
-
last_event.merge!(
|
74
|
-
"output" => output,
|
75
|
-
"updated_at" => Time.now,
|
76
|
-
"success" => true,
|
77
|
-
)
|
78
|
-
commit!
|
79
|
-
end
|
80
|
-
|
81
|
-
def fail_event!
|
82
|
-
fail_event
|
83
|
-
commit!
|
84
|
-
end
|
85
|
-
|
86
|
-
def fail_event
|
87
|
-
last_event["failed"] = true
|
65
|
+
subject.update_attributes!(field => store)
|
88
66
|
end
|
89
67
|
|
90
68
|
def status
|
91
69
|
store["status"]
|
92
70
|
end
|
93
71
|
|
94
|
-
def
|
95
|
-
"
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
if value
|
101
|
-
r.set(pipeline_status_key, value)
|
102
|
-
else
|
103
|
-
r.del(pipeline_status_key)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
store["pipeline_status"] = value
|
107
|
-
end
|
108
|
-
|
109
|
-
def commit_pipeline_status!(value = nil)
|
110
|
-
self.pipeline_status = value
|
111
|
-
last_event["cache"].clear
|
112
|
-
commit!
|
113
|
-
end
|
114
|
-
alias :reset_pipeline_status! :commit_pipeline_status!
|
115
|
-
|
116
|
-
def pipeline_status
|
117
|
-
DirtyPipeline.with_redis do |r|
|
118
|
-
store["pipeline_status"] = r.get(pipeline_status_key)
|
119
|
-
end
|
120
|
-
store.fetch("pipeline_status")
|
121
|
-
end
|
122
|
-
|
123
|
-
def state
|
124
|
-
store.fetch("state")
|
125
|
-
end
|
126
|
-
|
127
|
-
def events
|
128
|
-
store.fetch("events")
|
129
|
-
end
|
130
|
-
|
131
|
-
def last_event
|
132
|
-
events.last.to_h
|
133
|
-
end
|
134
|
-
|
135
|
-
def last_event_error(event_idx = nil)
|
136
|
-
event = events[event_idx] if event_idx
|
137
|
-
event ||= last_event
|
138
|
-
errors[event["error_idx"]].to_h
|
139
|
-
end
|
140
|
-
|
141
|
-
def errors
|
142
|
-
store.fetch("errors")
|
143
|
-
end
|
144
|
-
|
145
|
-
def last_error
|
146
|
-
errors.last.to_h
|
147
|
-
end
|
148
|
-
|
149
|
-
def reset_transaction_depth
|
150
|
-
store["transaction_depth"] = 1
|
151
|
-
end
|
152
|
-
|
153
|
-
def reset_transaction_depth!
|
154
|
-
reset_transaction_depth
|
155
|
-
commit!
|
156
|
-
end
|
157
|
-
|
158
|
-
def transaction_depth
|
159
|
-
store["transaction_depth"]
|
160
|
-
end
|
161
|
-
|
162
|
-
def increment_transaction_depth
|
163
|
-
store["transaction_depth"] = store["transaction_depth"].to_i + 1
|
164
|
-
end
|
165
|
-
|
166
|
-
def increment_transaction_depth!
|
167
|
-
increment_transaction_depth
|
168
|
-
commit!
|
169
|
-
end
|
170
|
-
|
171
|
-
def increment_attempts_count
|
172
|
-
last_event.merge!(
|
173
|
-
"attempts_count" => last_event["attempts_count"].to_i + 1
|
174
|
-
)
|
175
|
-
end
|
176
|
-
|
177
|
-
def increment_attempts_count!
|
178
|
-
increment_attempts_count
|
179
|
-
commit!
|
180
|
-
end
|
181
|
-
|
182
|
-
def save_retry(error)
|
183
|
-
save_error(error)
|
184
|
-
self.pipeline_status = RETRY_STATUS
|
185
|
-
end
|
186
|
-
|
187
|
-
def save_retry!(error)
|
188
|
-
save_retry(error)
|
189
|
-
commit!
|
190
|
-
end
|
191
|
-
|
192
|
-
def save_exception(exception)
|
193
|
-
errors << {
|
194
|
-
"error" => exception.class.to_s,
|
195
|
-
"error_message" => exception.message,
|
196
|
-
"created_at" => Time.current,
|
197
|
-
}
|
198
|
-
last_event["error_idx"] = errors.size - 1
|
199
|
-
fail_event
|
200
|
-
self.pipeline_status = FAILED_STATUS
|
201
|
-
end
|
202
|
-
|
203
|
-
def save_exception!(error)
|
204
|
-
save_exception(error)
|
205
|
-
commit!
|
206
|
-
end
|
207
|
-
|
208
|
-
def commit!
|
72
|
+
def commit!(event)
|
73
|
+
store["status"] = event.destination if event.destination
|
74
|
+
require'pry';binding.pry unless event.changes.respond_to?(:to_h)
|
75
|
+
store["state"].merge!(event.changes) unless event.changes.to_h.empty?
|
76
|
+
store["errors"][event.id] = event.error unless event.error.to_h.empty?
|
77
|
+
store["events"][event.id] = event.data unless event.data.to_h.empty?
|
209
78
|
subject.assign_attributes(field => store)
|
210
79
|
subject.save!
|
211
80
|
end
|
212
81
|
|
213
|
-
def
|
214
|
-
|
215
|
-
end
|
216
|
-
|
217
|
-
def failed?
|
218
|
-
pipeline_status == FAILED_STATUS
|
82
|
+
def processing_event
|
83
|
+
find_event(transactions_queue.processing_event.id)
|
219
84
|
end
|
220
85
|
|
221
|
-
def
|
222
|
-
[
|
86
|
+
def find_event(event_id)
|
87
|
+
return unless (found_event = store["events"][event_id])
|
88
|
+
Event.new(data: found_event, error: store["errors"][event_id])
|
223
89
|
end
|
224
90
|
end
|
225
91
|
end
|
@@ -1,76 +1,56 @@
|
|
1
1
|
module DirtyPipeline
|
2
2
|
class Transaction
|
3
|
-
attr_reader :locker, :storage, :subject, :pipeline
|
4
|
-
def initialize(pipeline)
|
3
|
+
attr_reader :locker, :storage, :subject, :pipeline, :queue, :event
|
4
|
+
def initialize(pipeline, queue, event)
|
5
5
|
@pipeline = pipeline
|
6
|
-
@storage = pipeline.storage
|
7
6
|
@subject = pipeline.subject
|
8
|
-
@
|
7
|
+
@storage = pipeline.storage
|
8
|
+
@queue = queue
|
9
|
+
@event = event
|
9
10
|
end
|
10
11
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
14
|
-
begin
|
15
|
-
destination, action, max_attempts_count =
|
16
|
-
find_transition(transition).values_at(:to, :action, :attempts)
|
12
|
+
def retry
|
13
|
+
event.attempt_retry!
|
14
|
+
pipeline.schedule_cleanup
|
17
15
|
|
18
|
-
|
19
|
-
subject.transaction(requires_new: true) do
|
20
|
-
raise ActiveRecord::Rollback if catch(:abort_transaction) do
|
21
|
-
yield(destination, action, *transition_args); nil
|
22
|
-
end
|
23
|
-
end
|
24
|
-
rescue => error
|
25
|
-
if try_again?(max_attempts_count)
|
26
|
-
Retry(error)
|
27
|
-
else
|
28
|
-
Exception(error)
|
29
|
-
end
|
30
|
-
raise
|
31
|
-
ensure
|
32
|
-
unless pipeline.status.success?
|
33
|
-
storage.events
|
34
|
-
.last(storage.transaction_depth)
|
35
|
-
.reverse
|
36
|
-
.each do |params|
|
37
|
-
transition = params["transition"]
|
38
|
-
targs = params["args"]
|
39
|
-
reversable_action = find_transition(transition).fetch(:action)
|
40
|
-
reversable_action.undo(self, *targs)
|
41
|
-
end
|
42
|
-
end
|
43
|
-
end
|
44
|
-
end
|
16
|
+
with_transaction { |*targs| yield(*targs) }
|
45
17
|
end
|
46
18
|
|
47
|
-
|
19
|
+
def call
|
20
|
+
# return unless queue.event_in_progress?(event)
|
48
21
|
|
49
|
-
|
50
|
-
|
51
|
-
pipeline.schedule_retry
|
52
|
-
end
|
22
|
+
event.start!
|
23
|
+
pipeline.schedule_cleanup
|
53
24
|
|
54
|
-
|
55
|
-
storage.save_exception!(error)
|
56
|
-
pipeline.status.error = error
|
57
|
-
pipeline.status.succeeded = false
|
25
|
+
with_transaction { |*targs| yield(*targs) }
|
58
26
|
end
|
59
27
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
28
|
+
private
|
29
|
+
|
30
|
+
def with_transaction
|
31
|
+
destination, action, max_attempts_count =
|
32
|
+
pipeline.find_transition(event.transition)
|
33
|
+
.values_at(:to, :action, :attempts)
|
64
34
|
|
65
|
-
|
66
|
-
|
67
|
-
|
35
|
+
storage.commit!(event)
|
36
|
+
|
37
|
+
# status.action_pool.unshift(action)
|
38
|
+
subject.transaction(requires_new: true) do
|
39
|
+
raise ActiveRecord::Rollback if catch(:abort_transaction) do
|
40
|
+
yield(destination, action, *event.args); nil
|
41
|
+
end
|
68
42
|
end
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
43
|
+
rescue => exception
|
44
|
+
event.link_exception(exception)
|
45
|
+
if max_attempts_count.to_i > event.attempts_count
|
46
|
+
event.retry!
|
47
|
+
pipeline.schedule_retry
|
48
|
+
else
|
49
|
+
pipeline.schedule_cleanup
|
73
50
|
end
|
51
|
+
raise
|
52
|
+
ensure
|
53
|
+
storage.commit!(event)
|
74
54
|
end
|
75
55
|
end
|
76
56
|
end
|
@@ -8,32 +8,41 @@ module DirtyPipeline
|
|
8
8
|
throw :fail_with_error, error
|
9
9
|
end
|
10
10
|
|
11
|
-
def Success(
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
def Success(changes = nil)
|
12
|
+
throw :success, changes.to_h
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.finalize(*args, **kwargs)
|
16
|
+
event, pipeline, *args = args
|
17
|
+
instance = new(event, *args, **kwargs)
|
18
|
+
pipeline.railway.switch_to(:call)
|
19
|
+
return unless instance.respond_to?(:finalize)
|
20
|
+
instance.finalize(pipeline.subject)
|
16
21
|
end
|
17
22
|
|
18
23
|
def self.undo(*args, **kwargs)
|
19
|
-
pipeline = args
|
20
|
-
instance = new(
|
24
|
+
event, pipeline, *args = args
|
25
|
+
instance = new(event, *args, **kwargs)
|
21
26
|
return unless instance.respond_to?(:undo)
|
22
27
|
instance.undo(pipeline.subject)
|
23
28
|
end
|
24
29
|
|
25
30
|
def self.call(*args, **kwargs)
|
26
|
-
pipeline = args
|
27
|
-
new(
|
31
|
+
event, pipeline, *args = args
|
32
|
+
instance = new(event, *args, **kwargs)
|
33
|
+
pipeline.railway[:undo] << event if instance.respond_to?(:undo)
|
34
|
+
pipeline.railway[:finalize] << event if instance.respond_to?(:finalize)
|
35
|
+
pipeline.railway.switch_to(:finalize) if instance.respond_to?(:finalize)
|
36
|
+
new(event, *args, **kwargs).call(pipeline.subject)
|
28
37
|
end
|
29
38
|
|
30
|
-
attr_reader :
|
31
|
-
def initialize(
|
32
|
-
@
|
39
|
+
attr_reader :event
|
40
|
+
def initialize(event, *, **)
|
41
|
+
@event = event
|
33
42
|
end
|
34
43
|
|
35
44
|
def fetch(key)
|
36
|
-
|
45
|
+
event.cache.fetch(key) { event.cache[key] = yield }
|
37
46
|
end
|
38
47
|
end
|
39
48
|
end
|
@@ -1,20 +1,27 @@
|
|
1
|
+
require 'sidekiq'
|
1
2
|
module DirtyPipeline
|
2
3
|
class Worker
|
3
4
|
include Sidekiq::Worker
|
5
|
+
using StringCamelcase
|
4
6
|
|
5
|
-
|
6
|
-
retry: 1,
|
7
|
-
dead: true
|
8
|
-
|
9
|
-
# args should contain - "enqueued_pipelines" - Array of Pipeline children
|
7
|
+
# args should contain - "enqueued_pipeline"
|
10
8
|
# args should contain - some args to find_subject
|
11
9
|
# args should contain - some args to make transition
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
pipeline_klass = options.fetch("enqueued_pipeline").constantize
|
10
|
+
def perform(options = {})
|
11
|
+
pipeline_klass =
|
12
|
+
Kernel.const_get(options.fetch("enqueued_pipeline").to_s.camelcase)
|
16
13
|
subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
|
17
|
-
|
14
|
+
transaction_id = options.fetch("transaction_id")
|
15
|
+
pipeline = pipeline_klass.new(subject, uuid: transaction_id)
|
16
|
+
operation = options.fetch("operation")
|
17
|
+
|
18
|
+
case operation
|
19
|
+
when "cleanup"
|
20
|
+
pipeline.clean
|
21
|
+
when "retry"
|
22
|
+
return pipeline.retry
|
23
|
+
end
|
24
|
+
pipeline.call
|
18
25
|
end
|
19
26
|
end
|
20
27
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dirty_pipeline
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.5.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Sergey Dolganov
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -72,7 +72,10 @@ files:
|
|
72
72
|
- dirty_pipeline.gemspec
|
73
73
|
- lib/dirty_pipeline.rb
|
74
74
|
- lib/dirty_pipeline/base.rb
|
75
|
-
- lib/dirty_pipeline/
|
75
|
+
- lib/dirty_pipeline/event.rb
|
76
|
+
- lib/dirty_pipeline/ext/camelcase.rb
|
77
|
+
- lib/dirty_pipeline/queue.rb
|
78
|
+
- lib/dirty_pipeline/railway.rb
|
76
79
|
- lib/dirty_pipeline/status.rb
|
77
80
|
- lib/dirty_pipeline/storage.rb
|
78
81
|
- lib/dirty_pipeline/transaction.rb
|
@@ -1,108 +0,0 @@
|
|
1
|
-
module DirtyPipeline
|
2
|
-
class Locker
|
3
|
-
CLEAN = "clean".freeze
|
4
|
-
|
5
|
-
attr_reader :storage, :subject
|
6
|
-
def initialize(subject, storage)
|
7
|
-
@subject = subject
|
8
|
-
@storage = storage
|
9
|
-
end
|
10
|
-
|
11
|
-
class Normal
|
12
|
-
attr_reader :locker, :transition, :transition_args
|
13
|
-
def initialize(locker, args)
|
14
|
-
@locker = locker
|
15
|
-
@transition = args.shift
|
16
|
-
@transition_args = args
|
17
|
-
end
|
18
|
-
|
19
|
-
# NORMAL MODE
|
20
|
-
# if state is PROCESSING_STATE - finish
|
21
|
-
# if state is FAILED_STATE - finish
|
22
|
-
# otherwise - start
|
23
|
-
def skip_any_action?
|
24
|
-
[
|
25
|
-
Storage::PROCESSING_STATUS,
|
26
|
-
Storage::FAILED_STATUS,
|
27
|
-
].include?(locker.storage.pipeline_status)
|
28
|
-
end
|
29
|
-
|
30
|
-
def start!
|
31
|
-
locker.storage.start!(transition, transition_args)
|
32
|
-
end
|
33
|
-
|
34
|
-
def lock!
|
35
|
-
return if skip_any_action?
|
36
|
-
start!
|
37
|
-
begin
|
38
|
-
yield(transition, *transition_args)
|
39
|
-
ensure
|
40
|
-
if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
|
41
|
-
locker.storage.reset_pipeline_status!
|
42
|
-
end
|
43
|
-
end
|
44
|
-
rescue
|
45
|
-
if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
|
46
|
-
locker.storage.commit_pipeline_status!(Storage::FAILED_STATUS)
|
47
|
-
end
|
48
|
-
raise
|
49
|
-
end
|
50
|
-
end
|
51
|
-
|
52
|
-
class Retry < Normal
|
53
|
-
def initialize(locker, _args)
|
54
|
-
@locker = locker
|
55
|
-
@transition = locker.storage.last_event["transition"]
|
56
|
-
@transition_args = locker.storage.last_event["input"]
|
57
|
-
end
|
58
|
-
|
59
|
-
def skip_any_action?
|
60
|
-
storage.status != Storage::RETRY_STATUS
|
61
|
-
end
|
62
|
-
|
63
|
-
def start!
|
64
|
-
locker.storage.start_retry!
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
# run in time much more then time to process an item
|
69
|
-
class Clean < Retry
|
70
|
-
ONE_DAY = 60 * 60 * 24
|
71
|
-
|
72
|
-
def skip_any_action?
|
73
|
-
return true if storage.status != Storage::PROCESSING_STATUS
|
74
|
-
started_less_then_a_day_ago?
|
75
|
-
end
|
76
|
-
|
77
|
-
def started_less_then_a_day_ago?
|
78
|
-
return unless (updated_at = locker.storage.last_event["updated_at"])
|
79
|
-
updated_at > one_day_ago
|
80
|
-
end
|
81
|
-
|
82
|
-
def one_day_ago
|
83
|
-
Time.now - ONE_DAY
|
84
|
-
end
|
85
|
-
end
|
86
|
-
|
87
|
-
def with_lock(*args)
|
88
|
-
lock!(*args) do |transition, *transition_args|
|
89
|
-
yield(transition, *transition_args)
|
90
|
-
end
|
91
|
-
end
|
92
|
-
|
93
|
-
def lock!(*args)
|
94
|
-
locker_klass(*args).new(self, args).lock! { |*largs| yield(*largs) }
|
95
|
-
end
|
96
|
-
|
97
|
-
def locker_klass(transition, *)
|
98
|
-
case transition
|
99
|
-
when Storage::RETRY_STATUS
|
100
|
-
Retry
|
101
|
-
when CLEAN
|
102
|
-
Clean
|
103
|
-
else
|
104
|
-
Normal
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
end
|