dirty_pipeline 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8c98084e4a9f05e60dbfb593de8661371a475d63d8831e80873eb0721b9b5fa6
4
- data.tar.gz: 445c7d82b17abac6ab4d9c795e1e715bd11ed3bcb944a42e9897ae9c95ef06b9
3
+ metadata.gz: 2d7bb36236434b64e26818af5a9ba0aa299035304fbbedc2dc305111b93b58a2
4
+ data.tar.gz: efed49a5b15c73610497df6ca1d6e05018efa586fbd8ae3d11df5f6092414fe6
5
5
  SHA512:
6
- metadata.gz: d1cec2be049b6a10eba0629d04287a685ffa6f00af52333647bc342dad67367a3de662b29d6db6e491b51dcbadf75f3d6b118fcf2bf3b5bf601c102fe522d051
7
- data.tar.gz: 90ac27df1e8166cc687f76e299350e0a069b983d72c039acd0a4d33881b2b1f9cc728bc0115f3a9af95a0cc781cfba94265838d55ebb8510a90f64e1a4252e0f
6
+ metadata.gz: c4324152fcc2d3e403ae93e23916fa2745820d199ed277cdbeef6d93b5d8dc726235a368de1c6e95fe6b124442c32e0b7e98fd531b74baef29b197c8f2bf702d
7
+ data.tar.gz: ec4cf7fbf3e02ec46bd89381f7534bde8a023cfa07f5cf0c3bfc5f3b040be49e7e4457ecfec6677fd01e7f2832b3012b7a824cf199acc94db5f6cb323d1fab13
@@ -1,13 +1,16 @@
1
1
  require "dirty_pipeline/version"
2
2
 
3
3
  module DirtyPipeline
4
- require_relative "dirty_pipeline/locker.rb"
5
- require_relative "dirty_pipeline/storage.rb"
6
- require_relative "dirty_pipeline/transition.rb"
4
+ require_relative "dirty_pipeline/ext/camelcase.rb"
7
5
  require_relative "dirty_pipeline/status.rb"
6
+ require_relative "dirty_pipeline/storage.rb"
8
7
  require_relative "dirty_pipeline/worker.rb"
9
8
  require_relative "dirty_pipeline/transaction.rb"
9
+ require_relative "dirty_pipeline/event.rb"
10
+ require_relative "dirty_pipeline/queue.rb"
11
+ require_relative "dirty_pipeline/railway.rb"
10
12
  require_relative "dirty_pipeline/base.rb"
13
+ require_relative "dirty_pipeline/transition.rb"
11
14
 
12
15
  # This method should yield raw Redis connection
13
16
  def self.with_redis
@@ -2,14 +2,7 @@ module DirtyPipeline
2
2
  class Base
3
3
  DEFAULT_RETRY_DELAY = 5 * 60 # 5 minutes
4
4
  DEFAULT_CLEANUP_DELAY = 60 * 60 * 24 # 1 day
5
- RESERVED_STATUSES = [
6
- Storage::FAILED_STATUS,
7
- Storage::PROCESSING_STATUS,
8
- Storage::RETRY_STATUS,
9
- Locker::CLEAN,
10
- ]
11
-
12
- class ReservedStatusError < StandardError; end
5
+
13
6
  class InvalidTransition < StandardError; end
14
7
 
15
8
  class << self
@@ -19,144 +12,153 @@ module DirtyPipeline
19
12
 
20
13
  attr_reader :transitions_map
21
14
  def inherited(child)
22
- child.instance_variable_set(:@transitions_map, Hash.new)
15
+ child.instance_variable_set(
16
+ :@transitions_map,
17
+ transitions_map || Hash.new
18
+ )
23
19
  end
24
- # PG JSONB column
25
- # {
26
- # status: :errored,
27
- # state: {
28
- # field: "value",
29
- # },
30
- # errors: [
31
- # {
32
- # error: "RuPost::API::Error",
33
- # error_message: "Timeout error",
34
- # created_at: 2018-01-01T13:22Z
35
- # },
36
- # ],
37
- # events: [
38
- # {
39
- # action: Init,
40
- # input: ...,
41
- # created_at: ...,
42
- # updated_at: ...,
43
- # attempts_count: 2,
44
- # },
45
- # {...},
46
- # ]
47
- # }
48
20
  attr_accessor :pipeline_storage, :retry_delay, :cleanup_delay
49
21
 
50
- def transition(action, from:, to:, name: action.to_s, attempts: 1)
51
- raise ReservedStatusError unless valid_statuses?(from, to)
52
- @transitions_map[name] = {
22
+ using StringCamelcase
23
+
24
+ def transition(name, from:, to:, action: nil, attempts: 1)
25
+ action ||= const_get(name.to_s.camelcase)
26
+ @transitions_map[name.to_s] = {
53
27
  action: action,
54
28
  from: Array(from).map(&:to_s),
55
29
  to: to.to_s,
56
30
  attempts: attempts,
57
31
  }
58
32
  end
59
-
60
- private
61
-
62
- def valid_statuses?(from, to)
63
- ((Array(to) + Array(from)) & RESERVED_STATUSES).empty?
64
- end
65
33
  end
66
34
 
67
- attr_reader :subject, :error, :storage, :status, :transitions_chain
68
- def initialize(subject)
35
+ attr_reader :subject, :storage, :status, :uuid, :queue, :railway
36
+ def initialize(subject, uuid: nil)
37
+ @uuid = uuid || Nanoid.generate
69
38
  @subject = subject
70
39
  @storage = Storage.new(subject, self.class.pipeline_storage)
71
- @status = Status.new(self)
72
- @transitions_chain = []
40
+ @railway = Railway.new(subject, @uuid)
41
+ @status = Status.success(subject)
73
42
  end
74
43
 
75
- def enqueue(transition_name, *args)
76
- DirtyPipeline::Worker.perform_async(
77
- "enqueued_pipeline" => self.class.to_s,
78
- "find_subject_args" => find_subject_args,
79
- "transition_args" => args.unshift(transition_name),
80
- )
44
+ def find_transition(name)
45
+ self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
46
+ next unless railway.operation.eql?(:call)
47
+ next if from == Array(storage.status)
48
+ next if from.include?(storage.status.to_s)
49
+ raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
50
+ end
81
51
  end
82
52
 
83
53
  def reset!
84
- storage.reset_pipeline_status!
54
+ railway.clear!
85
55
  end
86
56
 
87
57
  def clear!
88
58
  storage.clear!
89
- end
90
-
91
- def cache
92
- storage.last_event["cache"]
59
+ reset!
93
60
  end
94
61
 
95
62
  def chain(*args)
96
- transitions_chain << args
63
+ railway[:call] << Event.create(*args, tx_id: @uuid)
97
64
  self
98
65
  end
99
66
 
100
- def execute
101
- Result() do
102
- transitions_chain.each do |targs|
103
- call(*targs)
104
- storage.increment_transaction_depth!
105
- end
106
- storage.reset_transaction_depth!
107
- transitions_chain.clear
108
- end
67
+ def call
68
+ return self if (serialized_event = railway.next).nil?
69
+ execute(load_event(serialized_event))
109
70
  end
71
+ alias :call_next :call
110
72
 
111
- def call(*args)
112
- storage.reset_transaction_depth! if transitions_chain.empty?
113
- Result() do
114
- after_commit = nil
115
- # transaction with support of external calls
116
- transaction(*args) do |destination, action, *targs|
117
- output = {}
118
- fail_cause = nil
119
-
120
- output, *after_commit = catch(:success) do
121
- fail_cause = catch(:fail_with_error) do
122
- Abort() if catch(:abort) do
123
- throw :success, action.(self, *targs)
124
- end
125
- end
126
- nil
127
- end
73
+ def clean
74
+ railway.switch_to(:undo)
75
+ call_next
76
+ self
77
+ end
128
78
 
129
- if fail_cause
130
- Failure(fail_cause)
131
- else
132
- Success(destination, output)
133
- end
134
- end
79
+ def retry
80
+ return unless (event = load_event(railway.queue.processing_event))
81
+ execute(event, :retry)
82
+ end
135
83
 
136
- Array(after_commit).each { |cb| cb.call(subject) } if after_commit
137
- end
84
+ def schedule_cleanup
85
+ schedule("cleanup", cleanup_delay)
138
86
  end
139
87
 
140
88
  def schedule_retry
141
- ::DirtyPipeline::Worker.perform_in(
142
- retry_delay,
143
- "enqueued_pipeline" => self.class.to_s,
144
- "find_subject_args" => find_subject_args,
145
- "retry" => true,
146
- )
89
+ schedule("retry", retry_delay)
147
90
  end
148
91
 
149
- def schedule_cleanup
150
- ::DirtyPipeline::Worker.perform_in(
151
- cleanup_delay,
92
+ def schedule(operation, delay = nil)
93
+ job_args = {
94
+ "transaction_id" => @uuid,
152
95
  "enqueued_pipeline" => self.class.to_s,
153
96
  "find_subject_args" => find_subject_args,
154
- "transition_args" => [Locker::CLEAN],
155
- )
97
+ "operation" => operation,
98
+ }
99
+
100
+ if delay.nil?
101
+ ::DirtyPipeline::Worker.perform_async(job_args)
102
+ else
103
+ ::DirtyPipeline::Worker.perform_in(delay, job_args)
104
+ end
105
+ end
106
+
107
+ def when_success
108
+ yield(status.data, self) if status.success?
109
+ self
110
+ end
111
+
112
+ def when_failure(tag = status.tag)
113
+ yield(status.data, self) if status.failure? && status.tag == tag
114
+ self
156
115
  end
157
116
 
158
117
  private
159
118
 
119
+ def execute(event, type = :call)
120
+ transaction(event).public_send(type) do |destination, action, *args|
121
+ state_changes = process_action(action, event, *args)
122
+ next if status.failure?
123
+ Success(event, state_changes, destination)
124
+ end
125
+ call_next
126
+
127
+ self
128
+ end
129
+
130
+ def load_event(enqueued_event)
131
+ storage.find_event(enqueued_event.id) || enqueued_event
132
+ end
133
+
134
+ def process_action(action, event, *args)
135
+ return catch(:success) do
136
+ return if interupt_on_error(event) do
137
+ return if interupt_on_abort(event) do
138
+ throw :success, run_operation(action, event, *args); nil
139
+ end
140
+ end
141
+ nil
142
+ end
143
+ rescue => exception
144
+ @status = Status.failure(exception, tag: :exception)
145
+ raise
146
+ end
147
+
148
+ def run_operation(action, event, *args)
149
+ return unless action.respond_to?(operation = railway.operation)
150
+ action.public_send(operation, event, self, *args)
151
+ end
152
+
153
+ def interupt_on_error(event)
154
+ return if (fail_cause = catch(:fail_with_error) { yield; nil }).nil?
155
+ Failure(event, fail_cause)
156
+ end
157
+
158
+ def interupt_on_abort(event)
159
+ Abort(event) if catch(:abort) { yield; nil }
160
+ end
161
+
160
162
  def find_subject_args
161
163
  subject.id
162
164
  end
@@ -169,31 +171,27 @@ module DirtyPipeline
169
171
  self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
170
172
  end
171
173
 
172
- def transaction(*args)
173
- ::DirtyPipeline::Transaction.new(self).call(*args) do |*targs|
174
- yield(*targs)
175
- end
176
- end
177
-
178
- def Result()
179
- status.wrap { yield }
174
+ def transaction(event)
175
+ ::DirtyPipeline::Transaction.new(self, railway.queue, event)
180
176
  end
181
177
 
182
- def Failure(cause)
183
- storage.fail_event!
184
- status.error = cause
185
- status.succeeded = false
178
+ def Failure(event, cause)
179
+ event.failure!
180
+ railway.switch_to(:undo)
181
+ @status = Status.failure(cause, tag: :error)
182
+ throw :abort_transaction, true
186
183
  end
187
184
 
188
- def Abort()
189
- status.succeeded = false
185
+ def Abort(event)
186
+ event.failure!
187
+ railway.switch_to(:undo)
188
+ @status = Status.failure(subject, tag: :aborted)
190
189
  throw :abort_transaction, true
191
190
  end
192
191
 
193
- def Success(destination, output)
194
- cache.clear
195
- storage.complete!(output, destination)
196
- status.succeeded = true
192
+ def Success(event, changes, destination)
193
+ event.complete(changes, destination)
194
+ @status = Status.success(subject)
197
195
  end
198
196
  end
199
197
  end
@@ -0,0 +1,107 @@
1
+ require 'json'
2
+
3
+ module DirtyPipeline
4
+ class Event
5
+ NEW = "new".freeze
6
+ START = "started".freeze
7
+ FAILURE = "failed".freeze
8
+ RETRY = "retry".freeze
9
+ SUCCESS = "success".freeze
10
+
11
+ def self.create(transition, *args, tx_id:)
12
+ new(
13
+ data: {
14
+ "uuid" => Nanoid.generate,
15
+ "transaction_uuid" => tx_id,
16
+ "transition" => transition,
17
+ "args" => args,
18
+ }
19
+ )
20
+ end
21
+
22
+ def self.load(json)
23
+ return unless json
24
+ new(JSON.load(json))
25
+ end
26
+
27
+ def self.dump(event)
28
+ JSON.dump(event.to_h)
29
+ end
30
+
31
+ def dump
32
+ self.class.dump(self)
33
+ end
34
+
35
+ attr_reader :id, :tx_id, :error, :data
36
+ def initialize(options = {}, data: nil, error: nil)
37
+ unless options.empty?
38
+ options_hash = options.to_h
39
+ data ||= options_hash["data"]
40
+ error ||= options_hash["error"]
41
+ transition = options_hash["transition"]
42
+ args = options_hash["args"]
43
+ end
44
+
45
+ data_hash = data.to_h
46
+
47
+ @tx_id = data_hash.fetch("transaction_uuid")
48
+ @id = data_hash.fetch("uuid")
49
+ @data = {
50
+ "uuid" => @id,
51
+ "transaction_uuid" => @tx_id,
52
+ "transition" => transition,
53
+ "args" => args,
54
+ "created_at" => Time.now,
55
+ "cache" => {},
56
+ "attempts_count" => 1,
57
+ "status" => NEW,
58
+ }.merge(data_hash)
59
+ @error = error
60
+ end
61
+
62
+ def to_h
63
+ {data: @data, error: @error}
64
+ end
65
+
66
+ %w(args transition cache destination changes).each do |method_name|
67
+ define_method("#{method_name}") { @data[method_name] }
68
+ end
69
+
70
+ %w(new start retry failure).each do |method_name|
71
+ define_method("#{method_name}?") do
72
+ @data["status"] == self.class.const_get(method_name.upcase)
73
+ end
74
+
75
+ define_method("#{method_name}!") do
76
+ @data["status"] = self.class.const_get(method_name.upcase)
77
+ end
78
+ end
79
+
80
+ def link_exception(exception)
81
+ @error = {
82
+ "exception" => exception.class.to_s,
83
+ "exception_message" => exception.message,
84
+ "created_at" => Time.current,
85
+ }
86
+ failure!
87
+ end
88
+
89
+ def attempts_count
90
+ @data["attempts_count"].to_i
91
+ end
92
+
93
+ def attempt_retry
94
+ @data["updated_at"] = Time.now
95
+ @data["attempts_count"] = attempts_count + 1
96
+ end
97
+
98
+ def complete(changes, destination)
99
+ @data.merge!(
100
+ "destination" => destination,
101
+ "changes" => changes,
102
+ "updated_at" => Time.now,
103
+ "status" => SUCCESS,
104
+ )
105
+ end
106
+ end
107
+ end
@@ -0,0 +1,63 @@
1
+ module DirtyPipeline
2
+ module StringCamelcase
3
+ refine String do
4
+ # rubocop:disable Metrics/LineLength
5
+ # See gem Facets String#camelcase
6
+ #
7
+ # Converts a string to camelcase. This method leaves the first character
8
+ # as given. This allows other methods to be used first, such as #uppercase
9
+ # and #lowercase.
10
+ #
11
+ # "camel_case".camelcase #=> "camelCase"
12
+ # "Camel_case".camelcase #=> "CamelCase"
13
+ #
14
+ # Custom +separators+ can be used to specify the patterns used to determine
15
+ # where capitalization should occur. By default these are underscores (`_`)
16
+ # and space characters (`\s`).
17
+ #
18
+ # "camel/case".camelcase('/') #=> "camelCase"
19
+ #
20
+ # If the first separator is a symbol, either `:lower` or `:upper`, then
21
+ # the first characters of the string will be downcased or upcased respectively.
22
+ #
23
+ # "camel_case".camelcase(:upper) #=> "CamelCase"
24
+ #
25
+ # Note that this implementation is different from ActiveSupport's.
26
+ # If that is what you are looking for you may want {#modulize}.
27
+ #
28
+ # rubocop:enable Metrics/LineLength
29
+
30
+ # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
31
+ def camelcase(*separators)
32
+ case separators.first
33
+ when Symbol, TrueClass, FalseClass, NilClass
34
+ first_letter = separators.shift
35
+ end
36
+
37
+ separators = ["_", '\s'] if separators.empty?
38
+
39
+ str = dup
40
+
41
+ separators.each do |s|
42
+ str = str.gsub(/(?:#{s}+)([a-z])/) do
43
+ Regexp.last_match(1).upcase
44
+ end
45
+ end
46
+
47
+ case first_letter
48
+ when :upper, true
49
+ str = str.gsub(/(\A|\s)([a-z])/) do
50
+ Regexp.last_match(1) + Regexp.last_match(2).upcase
51
+ end
52
+ when :lower, false
53
+ str = str.gsub(/(\A|\s)([A-Z])/) do
54
+ Regexp.last_match(1) + Regexp.last_match(2).downcase
55
+ end
56
+ end
57
+
58
+ str
59
+ end
60
+ # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
61
+ end
62
+ end
63
+ end
@@ -0,0 +1,82 @@
1
+ module DirtyPipeline
2
+ class Queue
3
+ attr_reader :root
4
+ def initialize(operation, subject, transaction_id)
5
+ @root = "dirty-pipeline-queue:#{subject.class}:#{subject.id}:" \
6
+ "op_#{operation}:txid_#{transaction_id}"
7
+ end
8
+
9
+ def clear!
10
+ DirtyPipeline.with_redis do |r|
11
+ r.del active_event_key
12
+ r.del events_queue_key
13
+ end
14
+ end
15
+
16
+ def to_a
17
+ DirtyPipeline.with_redis { |r| r.lrange(events_queue_key, 0, -1) }
18
+ end
19
+
20
+ def push(event)
21
+ DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
22
+ end
23
+ alias :<< :push
24
+
25
+ def unshift(event)
26
+ DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
27
+ end
28
+
29
+ def dequeue
30
+ DirtyPipeline.with_redis do |r|
31
+ data = r.lpop(events_queue_key)
32
+ data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
33
+ return unpack(data)
34
+ end
35
+ end
36
+ alias :pop :dequeue
37
+
38
+ def event_in_progress?(event = nil)
39
+ if event.nil?
40
+ !processing_event.nil?
41
+ else
42
+ processing_event.id == event.id
43
+ end
44
+ end
45
+
46
+ def processing_event
47
+ DirtyPipeline.with_redis { |r| unpack r.get(active_event_key) }
48
+ end
49
+
50
+ private
51
+
52
+ def pack(event)
53
+ JSON.dump(
54
+ "evid" => event.id,
55
+ "txid" => event.tx_id,
56
+ "transit" => event.transition,
57
+ "args" => event.args,
58
+ )
59
+ end
60
+
61
+ def unpack(packed_event)
62
+ return unless packed_event
63
+ unpacked_event = JSON.load(packed_event)
64
+ Event.new(
65
+ data: {
66
+ "uuid" => unpacked_event["evid"],
67
+ "transaction_uuid" => unpacked_event["txid"],
68
+ "transition" => unpacked_event["transit"],
69
+ "args" => unpacked_event["args"],
70
+ }
71
+ )
72
+ end
73
+
74
+ def events_queue_key
75
+ "#{root}:events"
76
+ end
77
+
78
+ def active_event_key
79
+ "#{root}:active"
80
+ end
81
+ end
82
+ end
@@ -0,0 +1,74 @@
1
+ module DirtyPipeline
2
+ class Railway
3
+ OPERATIONS = %w(call undo finalize)
4
+
5
+ def initialize(subject, transaction_id)
6
+ @tx_id = transaction_id
7
+ @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:" \
8
+ ":txid_#{transaction_id}"
9
+ @queues = Hash[
10
+ OPERATIONS.map do |operation|
11
+ [operation, Queue.new(operation, subject, transaction_id)]
12
+ end
13
+ ]
14
+ end
15
+
16
+ def clear!
17
+ @queues.values.each(&:clear!)
18
+ DirtyPipeline.with_redis { |r| r.del(active_operation_key) }
19
+ end
20
+
21
+ def next
22
+ return if other_transaction_in_progress?
23
+ start_transaction! if running_transaction.nil?
24
+
25
+ queue.pop.tap { |event| finish_transaction! if event.nil? }
26
+ end
27
+
28
+ def queue(name = active)
29
+ @queues[name.to_s]
30
+ end
31
+ alias :[] :queue
32
+
33
+ def switch_to(name)
34
+ raise ArgumentError unless OPERATIONS.include?(name.to_s)
35
+ return if name.to_s == active
36
+ DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
37
+ end
38
+
39
+ def active
40
+ DirtyPipeline.with_redis { |r| r.get(active_operation_key) }
41
+ end
42
+ alias :operation :active
43
+
44
+ private
45
+
46
+ def active_transaction_key
47
+ "#{@root}:active_transaction"
48
+ end
49
+
50
+ def active_operation_key
51
+ "#{@root}:active_operation"
52
+ end
53
+
54
+ def start_transaction!
55
+ switch_to(OPERATIONS.first)
56
+ DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
57
+ end
58
+
59
+ def finish_transaction!
60
+ return unless running_transaction == @tx_id
61
+ DirtyPipeline.with_redis { |r| r.del(active_transaction_key) }
62
+ @queues.values.each(&:clear!)
63
+ end
64
+
65
+ def running_transaction
66
+ DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
67
+ end
68
+
69
+ def other_transaction_in_progress?
70
+ return false if running_transaction.nil?
71
+ running_transaction != @tx_id
72
+ end
73
+ end
74
+ end
@@ -1,55 +1,27 @@
1
1
  module DirtyPipeline
2
- class Status < SimpleDelegator
3
- attr_accessor :error, :succeeded, :action_pool
4
- attr_reader :storage, :pipeline
5
- def initialize(*)
6
- super
7
- @storage = __getobj__.storage
8
- @action_pool = []
9
- end
2
+ class Status
3
+ attr_reader :success, :tag, :data
10
4
 
11
- def wrap
12
- return self if succeeded == false
13
- self.succeeded = nil
14
- yield
15
- self
5
+ def self.success(data, tag: :success)
6
+ new(true, data, tag)
16
7
  end
17
8
 
18
- def success?
19
- !!succeeded
9
+ def self.failure(data, tag: :exception)
10
+ new(false, data, tag)
20
11
  end
21
12
 
22
- def when_success(callback = nil)
23
- return self unless success?
24
- block_given? ? yield(self) : callback.(self)
25
- self
13
+ def initialize(success, data, tag = nil)
14
+ @success = success
15
+ @data = data
16
+ @tag = tag
26
17
  end
27
18
 
28
- def when_failed(callback = nil)
29
- return self unless storage.failed?
30
- block_given? ? yield(self) : callback.(self)
31
- self
32
- end
33
-
34
- def errored?
35
- return if succeeded.nil?
36
- ready? && !succeeded
37
- end
38
-
39
- def when_error(callback = nil)
40
- return self unless errored?
41
- block_given? ? yield(self) : callback.(self)
42
- self
43
- end
44
-
45
- def ready?
46
- storage.pipeline_status.nil?
19
+ def success?
20
+ !!success
47
21
  end
48
22
 
49
- def when_processing(callback = nil)
50
- return self unless storage.processing?
51
- block_given? ? yield(self) : callback.(self)
52
- self
23
+ def failure?
24
+ !success?
53
25
  end
54
26
  end
55
27
  end
@@ -1,11 +1,12 @@
1
1
  module DirtyPipeline
2
2
  class Storage
3
- FAILED_STATUS = "failed".freeze
3
+ SUCCESS_STATUS = "success".freeze
4
+ FAILURE_STATUS = "failure".freeze
4
5
  RETRY_STATUS = "retry".freeze
5
6
  PROCESSING_STATUS = "processing".freeze
6
7
  class InvalidPipelineStorage < StandardError; end
7
8
 
8
- attr_reader :subject, :field
9
+ attr_reader :subject, :field, :transactions_queue
9
10
  attr_accessor :store
10
11
  alias :to_h :store
11
12
  def initialize(subject, field)
@@ -16,210 +17,75 @@ module DirtyPipeline
16
17
 
17
18
  def init_store(store_field)
18
19
  self.store = subject.send(store_field).to_h
19
- clear! if store.empty?
20
+ clear if store.empty?
20
21
  return if valid_store?
21
22
  raise InvalidPipelineStorage, store
22
23
  end
23
24
 
24
25
  def valid_store?
25
- (
26
- store.keys &
27
- %w(status pipeline_status events errors state transaction_depth)
28
- ).size == 6
29
- end
30
-
26
+ (store.keys & %w(status events errors state)).size.eql?(4)
27
+ end
28
+
29
+ # PG JSONB column
30
+ # {
31
+ # status: :errored,
32
+ # state: {
33
+ # field: "value",
34
+ # },
35
+ # errors: {
36
+ # "<event_id>": {
37
+ # error: "RuPost::API::Error",
38
+ # error_message: "Timeout error",
39
+ # created_at: 2018-01-01T13:22Z
40
+ # },
41
+ # },
42
+ # events: {
43
+ # <event_id>: {
44
+ # action: Init,
45
+ # input: ...,
46
+ # created_at: ...,
47
+ # updated_at: ...,
48
+ # attempts_count: 2,
49
+ # },
50
+ # <event_id>: {...},
51
+ # }
52
+ # }
31
53
  def clear
32
54
  self.store = subject.send(
33
55
  "#{field}=",
34
56
  "status" => nil,
35
- "pipeline_status" => nil,
36
57
  "state" => {},
37
- "events" => [],
38
- "errors" => [],
39
- "transaction_depth" => 1
58
+ "events" => {},
59
+ "errors" => {}
40
60
  )
41
- DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
42
61
  end
43
62
 
44
63
  def clear!
45
64
  clear
46
- commit!
47
- end
48
-
49
- def start!(transition, args)
50
- events << {
51
- "transition" => transition,
52
- "args" => args,
53
- "created_at" => Time.now,
54
- "cache" => {},
55
- }
56
- increment_attempts_count
57
- self.pipeline_status = PROCESSING_STATUS
58
- # self.status = "processing", should be set by Locker
59
- commit!
60
- end
61
-
62
- def start_retry!
63
- last_event.merge!(updated_at: Time.now)
64
- increment_attempts_count
65
- self.pipeline_status = PROCESSING_STATUS
66
- # self.status = "processing", should be set by Locker
67
- commit!
68
- end
69
-
70
- def complete!(output, destination)
71
- store["status"] = destination
72
- state.merge!(output)
73
- last_event.merge!(
74
- "output" => output,
75
- "updated_at" => Time.now,
76
- "success" => true,
77
- )
78
- commit!
79
- end
80
-
81
- def fail_event!
82
- fail_event
83
- commit!
84
- end
85
-
86
- def fail_event
87
- last_event["failed"] = true
65
+ subject.update_attributes!(field => store)
88
66
  end
89
67
 
90
68
  def status
91
69
  store["status"]
92
70
  end
93
71
 
94
- def pipeline_status_key
95
- "pipeline-status:#{subject.class}:#{subject.id}:#{field}"
96
- end
97
-
98
- def pipeline_status=(value)
99
- DirtyPipeline.with_redis do |r|
100
- if value
101
- r.set(pipeline_status_key, value)
102
- else
103
- r.del(pipeline_status_key)
104
- end
105
- end
106
- store["pipeline_status"] = value
107
- end
108
-
109
- def commit_pipeline_status!(value = nil)
110
- self.pipeline_status = value
111
- last_event["cache"].clear
112
- commit!
113
- end
114
- alias :reset_pipeline_status! :commit_pipeline_status!
115
-
116
- def pipeline_status
117
- DirtyPipeline.with_redis do |r|
118
- store["pipeline_status"] = r.get(pipeline_status_key)
119
- end
120
- store.fetch("pipeline_status")
121
- end
122
-
123
- def state
124
- store.fetch("state")
125
- end
126
-
127
- def events
128
- store.fetch("events")
129
- end
130
-
131
- def last_event
132
- events.last.to_h
133
- end
134
-
135
- def last_event_error(event_idx = nil)
136
- event = events[event_idx] if event_idx
137
- event ||= last_event
138
- errors[event["error_idx"]].to_h
139
- end
140
-
141
- def errors
142
- store.fetch("errors")
143
- end
144
-
145
- def last_error
146
- errors.last.to_h
147
- end
148
-
149
- def reset_transaction_depth
150
- store["transaction_depth"] = 1
151
- end
152
-
153
- def reset_transaction_depth!
154
- reset_transaction_depth
155
- commit!
156
- end
157
-
158
- def transaction_depth
159
- store["transaction_depth"]
160
- end
161
-
162
- def increment_transaction_depth
163
- store["transaction_depth"] = store["transaction_depth"].to_i + 1
164
- end
165
-
166
- def increment_transaction_depth!
167
- increment_transaction_depth
168
- commit!
169
- end
170
-
171
- def increment_attempts_count
172
- last_event.merge!(
173
- "attempts_count" => last_event["attempts_count"].to_i + 1
174
- )
175
- end
176
-
177
- def increment_attempts_count!
178
- increment_attempts_count
179
- commit!
180
- end
181
-
182
- def save_retry(error)
183
- save_error(error)
184
- self.pipeline_status = RETRY_STATUS
185
- end
186
-
187
- def save_retry!(error)
188
- save_retry(error)
189
- commit!
190
- end
191
-
192
- def save_exception(exception)
193
- errors << {
194
- "error" => exception.class.to_s,
195
- "error_message" => exception.message,
196
- "created_at" => Time.current,
197
- }
198
- last_event["error_idx"] = errors.size - 1
199
- fail_event
200
- self.pipeline_status = FAILED_STATUS
201
- end
202
-
203
- def save_exception!(error)
204
- save_exception(error)
205
- commit!
206
- end
207
-
208
- def commit!
72
+ def commit!(event)
73
+ store["status"] = event.destination if event.destination
74
+ require'pry';binding.pry unless event.changes.respond_to?(:to_h)
75
+ store["state"].merge!(event.changes) unless event.changes.to_h.empty?
76
+ store["errors"][event.id] = event.error unless event.error.to_h.empty?
77
+ store["events"][event.id] = event.data unless event.data.to_h.empty?
209
78
  subject.assign_attributes(field => store)
210
79
  subject.save!
211
80
  end
212
81
 
213
- def ready?
214
- storage.pipeline_status.nil?
215
- end
216
-
217
- def failed?
218
- pipeline_status == FAILED_STATUS
82
+ def processing_event
83
+ find_event(transactions_queue.processing_event.id)
219
84
  end
220
85
 
221
- def processing?
222
- [PROCESSING_STATUS, RETRY_STATUS].include?(pipeline_status)
86
+ def find_event(event_id)
87
+ return unless (found_event = store["events"][event_id])
88
+ Event.new(data: found_event, error: store["errors"][event_id])
223
89
  end
224
90
  end
225
91
  end
@@ -1,76 +1,56 @@
1
1
  module DirtyPipeline
2
2
  class Transaction
3
- attr_reader :locker, :storage, :subject, :pipeline
4
- def initialize(pipeline)
3
+ attr_reader :locker, :storage, :subject, :pipeline, :queue, :event
4
+ def initialize(pipeline, queue, event)
5
5
  @pipeline = pipeline
6
- @storage = pipeline.storage
7
6
  @subject = pipeline.subject
8
- @locker = Locker.new(@subject, @storage)
7
+ @storage = pipeline.storage
8
+ @queue = queue
9
+ @event = event
9
10
  end
10
11
 
11
- def call(*args)
12
- locker.with_lock(*args) do |transition, *transition_args|
13
- pipeline.schedule_cleanup
14
- begin
15
- destination, action, max_attempts_count =
16
- find_transition(transition).values_at(:to, :action, :attempts)
12
+ def retry
13
+ event.attempt_retry!
14
+ pipeline.schedule_cleanup
17
15
 
18
- # status.action_pool.unshift(action)
19
- subject.transaction(requires_new: true) do
20
- raise ActiveRecord::Rollback if catch(:abort_transaction) do
21
- yield(destination, action, *transition_args); nil
22
- end
23
- end
24
- rescue => error
25
- if try_again?(max_attempts_count)
26
- Retry(error)
27
- else
28
- Exception(error)
29
- end
30
- raise
31
- ensure
32
- unless pipeline.status.success?
33
- storage.events
34
- .last(storage.transaction_depth)
35
- .reverse
36
- .each do |params|
37
- transition = params["transition"]
38
- targs = params["args"]
39
- reversable_action = find_transition(transition).fetch(:action)
40
- reversable_action.undo(self, *targs)
41
- end
42
- end
43
- end
44
- end
16
+ with_transaction { |*targs| yield(*targs) }
45
17
  end
46
18
 
47
- private
19
+ def call
20
+ # return unless queue.event_in_progress?(event)
48
21
 
49
- def Retry(error, *args)
50
- storage.save_retry!(error)
51
- pipeline.schedule_retry
52
- end
22
+ event.start!
23
+ pipeline.schedule_cleanup
53
24
 
54
- def Exception(error)
55
- storage.save_exception!(error)
56
- pipeline.status.error = error
57
- pipeline.status.succeeded = false
25
+ with_transaction { |*targs| yield(*targs) }
58
26
  end
59
27
 
60
- def try_again?(max_attempts_count)
61
- return false unless max_attempts_count
62
- storage.last_event["attempts_count"].to_i < max_attempts_count
63
- end
28
+ private
29
+
30
+ def with_transaction
31
+ destination, action, max_attempts_count =
32
+ pipeline.find_transition(event.transition)
33
+ .values_at(:to, :action, :attempts)
64
34
 
65
- def find_transition(name)
66
- if (const_name = pipeline.class.const_get(name) rescue nil)
67
- name = const_name.to_s
35
+ storage.commit!(event)
36
+
37
+ # status.action_pool.unshift(action)
38
+ subject.transaction(requires_new: true) do
39
+ raise ActiveRecord::Rollback if catch(:abort_transaction) do
40
+ yield(destination, action, *event.args); nil
41
+ end
68
42
  end
69
- pipeline.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
70
- next if from == Array(storage.status)
71
- next if from.include?(storage.status.to_s)
72
- raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
43
+ rescue => exception
44
+ event.link_exception(exception)
45
+ if max_attempts_count.to_i > event.attempts_count
46
+ event.retry!
47
+ pipeline.schedule_retry
48
+ else
49
+ pipeline.schedule_cleanup
73
50
  end
51
+ raise
52
+ ensure
53
+ storage.commit!(event)
74
54
  end
75
55
  end
76
56
  end
@@ -8,32 +8,41 @@ module DirtyPipeline
8
8
  throw :fail_with_error, error
9
9
  end
10
10
 
11
- def Success(output = nil, after_commit: nil, &block)
12
- result = [output.to_h]
13
- after_commit = Array(after_commit) << block if block_given?
14
- result += Array(after_commit) if after_commit
15
- throw :success, result
11
+ def Success(changes = nil)
12
+ throw :success, changes.to_h
13
+ end
14
+
15
+ def self.finalize(*args, **kwargs)
16
+ event, pipeline, *args = args
17
+ instance = new(event, *args, **kwargs)
18
+ pipeline.railway.switch_to(:call)
19
+ return unless instance.respond_to?(:finalize)
20
+ instance.finalize(pipeline.subject)
16
21
  end
17
22
 
18
23
  def self.undo(*args, **kwargs)
19
- pipeline = args.shift
20
- instance = new(pipeline, *args, **kwargs)
24
+ event, pipeline, *args = args
25
+ instance = new(event, *args, **kwargs)
21
26
  return unless instance.respond_to?(:undo)
22
27
  instance.undo(pipeline.subject)
23
28
  end
24
29
 
25
30
  def self.call(*args, **kwargs)
26
- pipeline = args.shift
27
- new(pipeline, *args, **kwargs).call(pipeline.subject)
31
+ event, pipeline, *args = args
32
+ instance = new(event, *args, **kwargs)
33
+ pipeline.railway[:undo] << event if instance.respond_to?(:undo)
34
+ pipeline.railway[:finalize] << event if instance.respond_to?(:finalize)
35
+ pipeline.railway.switch_to(:finalize) if instance.respond_to?(:finalize)
36
+ new(event, *args, **kwargs).call(pipeline.subject)
28
37
  end
29
38
 
30
- attr_reader :pipeline
31
- def initialize(pipeline, *, **)
32
- @pipeline = pipeline
39
+ attr_reader :event
40
+ def initialize(event, *, **)
41
+ @event = event
33
42
  end
34
43
 
35
44
  def fetch(key)
36
- pipeline.cache.fetch(key) { pipeline.cache[key] = yield }
45
+ event.cache.fetch(key) { event.cache[key] = yield }
37
46
  end
38
47
  end
39
48
  end
@@ -1,3 +1,3 @@
1
1
  module DirtyPipeline
2
- VERSION = "0.4.0"
2
+ VERSION = "0.5.0"
3
3
  end
@@ -1,20 +1,27 @@
1
+ require 'sidekiq'
1
2
  module DirtyPipeline
2
3
  class Worker
3
4
  include Sidekiq::Worker
5
+ using StringCamelcase
4
6
 
5
- sidekiq_options queue: "default",
6
- retry: 1,
7
- dead: true
8
-
9
- # args should contain - "enqueued_pipelines" - Array of Pipeline children
7
+ # args should contain - "enqueued_pipeline"
10
8
  # args should contain - some args to find_subject
11
9
  # args should contain - some args to make transition
12
-
13
- def perform(options)
14
- # FIXME: get rid of ActiveSupport #constantize
15
- pipeline_klass = options.fetch("enqueued_pipeline").constantize
10
+ def perform(options = {})
11
+ pipeline_klass =
12
+ Kernel.const_get(options.fetch("enqueued_pipeline").to_s.camelcase)
16
13
  subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
17
- pipeline_klass.new(subject).call(*options.fetch("transition_args"))
14
+ transaction_id = options.fetch("transaction_id")
15
+ pipeline = pipeline_klass.new(subject, uuid: transaction_id)
16
+ operation = options.fetch("operation")
17
+
18
+ case operation
19
+ when "cleanup"
20
+ pipeline.clean
21
+ when "retry"
22
+ return pipeline.retry
23
+ end
24
+ pipeline.call
18
25
  end
19
26
  end
20
27
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: dirty_pipeline
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.5.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Sergey Dolganov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-08-25 00:00:00.000000000 Z
11
+ date: 2018-09-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -72,7 +72,10 @@ files:
72
72
  - dirty_pipeline.gemspec
73
73
  - lib/dirty_pipeline.rb
74
74
  - lib/dirty_pipeline/base.rb
75
- - lib/dirty_pipeline/locker.rb
75
+ - lib/dirty_pipeline/event.rb
76
+ - lib/dirty_pipeline/ext/camelcase.rb
77
+ - lib/dirty_pipeline/queue.rb
78
+ - lib/dirty_pipeline/railway.rb
76
79
  - lib/dirty_pipeline/status.rb
77
80
  - lib/dirty_pipeline/storage.rb
78
81
  - lib/dirty_pipeline/transaction.rb
@@ -1,108 +0,0 @@
1
- module DirtyPipeline
2
- class Locker
3
- CLEAN = "clean".freeze
4
-
5
- attr_reader :storage, :subject
6
- def initialize(subject, storage)
7
- @subject = subject
8
- @storage = storage
9
- end
10
-
11
- class Normal
12
- attr_reader :locker, :transition, :transition_args
13
- def initialize(locker, args)
14
- @locker = locker
15
- @transition = args.shift
16
- @transition_args = args
17
- end
18
-
19
- # NORMAL MODE
20
- # if state is PROCESSING_STATE - finish
21
- # if state is FAILED_STATE - finish
22
- # otherwise - start
23
- def skip_any_action?
24
- [
25
- Storage::PROCESSING_STATUS,
26
- Storage::FAILED_STATUS,
27
- ].include?(locker.storage.pipeline_status)
28
- end
29
-
30
- def start!
31
- locker.storage.start!(transition, transition_args)
32
- end
33
-
34
- def lock!
35
- return if skip_any_action?
36
- start!
37
- begin
38
- yield(transition, *transition_args)
39
- ensure
40
- if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
41
- locker.storage.reset_pipeline_status!
42
- end
43
- end
44
- rescue
45
- if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
46
- locker.storage.commit_pipeline_status!(Storage::FAILED_STATUS)
47
- end
48
- raise
49
- end
50
- end
51
-
52
- class Retry < Normal
53
- def initialize(locker, _args)
54
- @locker = locker
55
- @transition = locker.storage.last_event["transition"]
56
- @transition_args = locker.storage.last_event["input"]
57
- end
58
-
59
- def skip_any_action?
60
- storage.status != Storage::RETRY_STATUS
61
- end
62
-
63
- def start!
64
- locker.storage.start_retry!
65
- end
66
- end
67
-
68
- # run in time much more then time to process an item
69
- class Clean < Retry
70
- ONE_DAY = 60 * 60 * 24
71
-
72
- def skip_any_action?
73
- return true if storage.status != Storage::PROCESSING_STATUS
74
- started_less_then_a_day_ago?
75
- end
76
-
77
- def started_less_then_a_day_ago?
78
- return unless (updated_at = locker.storage.last_event["updated_at"])
79
- updated_at > one_day_ago
80
- end
81
-
82
- def one_day_ago
83
- Time.now - ONE_DAY
84
- end
85
- end
86
-
87
- def with_lock(*args)
88
- lock!(*args) do |transition, *transition_args|
89
- yield(transition, *transition_args)
90
- end
91
- end
92
-
93
- def lock!(*args)
94
- locker_klass(*args).new(self, args).lock! { |*largs| yield(*largs) }
95
- end
96
-
97
- def locker_klass(transition, *)
98
- case transition
99
- when Storage::RETRY_STATUS
100
- Retry
101
- when CLEAN
102
- Clean
103
- else
104
- Normal
105
- end
106
- end
107
- end
108
- end