dirty_pipeline 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6b8f5e951d183bae44a012daeccb345d84b0ccc0dd26edeb1006cfe67cca27ba
-  data.tar.gz: 2b166ca245ea06ede75099e5c511749e9b9af04414446948e81db9ad5ce2b485
+  metadata.gz: 8c98084e4a9f05e60dbfb593de8661371a475d63d8831e80873eb0721b9b5fa6
+  data.tar.gz: 445c7d82b17abac6ab4d9c795e1e715bd11ed3bcb944a42e9897ae9c95ef06b9
 SHA512:
-  metadata.gz: 0ca3237ae6a6f282040216add989af7a84201d9d1a09be280d177c83de6ff752d55e7d5b3394c1f1e887677ec3e6280feff90f450df5ae18a550873f49bd80b9
-  data.tar.gz: 0c45a1200ffa8074fda41c1cf7839444d2f72d74a3bf1d4959f8f2a599ffac6aff0eed0a00f973799e4a99f4ef7e4f0fc983301bd39eba0306386592f651aace
+  metadata.gz: d1cec2be049b6a10eba0629d04287a685ffa6f00af52333647bc342dad67367a3de662b29d6db6e491b51dcbadf75f3d6b118fcf2bf3b5bf601c102fe522d051
+  data.tar.gz: 90ac27df1e8166cc687f76e299350e0a069b983d72c039acd0a4d33881b2b1f9cc728bc0115f3a9af95a0cc781cfba94265838d55ebb8510a90f64e1a4252e0f

data/lib/dirty_pipeline.rb

@@ -5,6 +5,8 @@ module DirtyPipeline
   require_relative "dirty_pipeline/storage.rb"
   require_relative "dirty_pipeline/transition.rb"
   require_relative "dirty_pipeline/status.rb"
+  require_relative "dirty_pipeline/worker.rb"
+  require_relative "dirty_pipeline/transaction.rb"
   require_relative "dirty_pipeline/base.rb"
 
   # This method should yield raw Redis connection

data/lib/dirty_pipeline/base.rb

@@ -64,8 +64,16 @@ module DirtyPipeline
       end
     end
 
+    attr_reader :subject, :error, :storage, :status, :transitions_chain
+    def initialize(subject)
+      @subject = subject
+      @storage = Storage.new(subject, self.class.pipeline_storage)
+      @status = Status.new(self)
+      @transitions_chain = []
+    end
+
     def enqueue(transition_name, *args)
-      Shipping::PipelineWorker.perform_async(
+      DirtyPipeline::Worker.perform_async(
         "enqueued_pipeline" => self.class.to_s,
         "find_subject_args" => find_subject_args,
         "transition_args" => args.unshift(transition_name),
@@ -81,18 +89,27 @@ module DirtyPipeline
     end
 
     def cache
-      storage.store["cache"]
+      storage.last_event["cache"]
     end
 
-    attr_reader :subject, :error, :storage, :status
-    def initialize(subject)
-      @subject = subject
-      @storage = Storage.new(subject, self.class.pipeline_storage)
-      @locker = Locker.new(@subject, @storage)
-      @status = Status.new(self)
+    def chain(*args)
+      transitions_chain << args
+      self
+    end
+
+    def execute
+      Result() do
+        transitions_chain.each do |targs|
+          call(*targs)
+          storage.increment_transaction_depth!
+        end
+        storage.reset_transaction_depth!
+        transitions_chain.clear
+      end
     end
 
     def call(*args)
+      storage.reset_transaction_depth! if transitions_chain.empty?
       Result() do
         after_commit = nil
         # transaction with support of external calls
@@ -110,7 +127,7 @@ module DirtyPipeline
           end
 
           if fail_cause
-            ExpectedError(fail_cause)
+            Failure(fail_cause)
           else
             Success(destination, output)
           end
@@ -120,9 +137,25 @@ module DirtyPipeline
       end
     end
 
-    private
+    def schedule_retry
+      ::DirtyPipeline::Worker.perform_in(
+        retry_delay,
+        "enqueued_pipeline" => self.class.to_s,
+        "find_subject_args" => find_subject_args,
+        "retry" => true,
+      )
+    end
+
+    def schedule_cleanup
+      ::DirtyPipeline::Worker.perform_in(
+        cleanup_delay,
+        "enqueued_pipeline" => self.class.to_s,
+        "find_subject_args" => find_subject_args,
+        "transition_args" => [Locker::CLEAN],
+      )
+    end
 
-    attr_reader :locker
+    private
 
     def find_subject_args
       subject.id
@@ -136,29 +169,19 @@ module DirtyPipeline
       self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
     end
 
-    def Result()
-      status.wrap { yield }
+    def transaction(*args)
+      ::DirtyPipeline::Transaction.new(self).call(*args) do |*targs|
+        yield(*targs)
+      end
     end
 
-    def Retry(error, *args)
-      storage.save_retry!(error)
-      Shipping::PipelineWorker.perform_in(
-        retry_delay,
-        "enqueued_pipeline" => self.class.to_s,
-        "find_subject_args" => find_subject_args,
-        "retry" => true,
-      )
+    def Result()
+      status.wrap { yield }
     end
 
-    def ExpectedError(cause)
-      status.error = cause
+    def Failure(cause)
       storage.fail_event!
-      status.succeeded = false
-    end
-
-    def Exception(error)
-      storage.save_exception!(error)
-      status.error = error
+      status.error = cause
       status.succeeded = false
     end
 
@@ -172,61 +195,5 @@ module DirtyPipeline
       storage.complete!(output, destination)
       status.succeeded = true
     end
-
-    def try_again?(max_attempts_count)
-      return unless max_attempts_count
-      storage.last_event["attempts_count"].to_i < max_attempts_count
-    end
-
-    def find_transition(name)
-      if (const_name = self.class.const_get(name) rescue nil)
-        name = const_name.to_s
-      end
-      self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
-        next if from == Array(storage.status)
-        next if from.include?(storage.status.to_s)
-        raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
-      end
-    end
-
-    def schedule_cleanup
-      Shipping::PipelineWorker.perform_in(
-        cleanup_delay,
-        "enqueued_pipeline" => self.class.to_s,
-        "find_subject_args" => find_subject_args,
-        "transition_args" => [Locker::CLEAN],
-      )
-    end
-
-    def transaction(*args)
-      locker.with_lock(*args) do |transition, *transition_args|
-        begin
-          schedule_cleanup
-          destination, action, max_attempts_count =
-            find_transition(transition).values_at(:to, :action, :attempts)
-
-          status.action_pool.unshift(action)
-          subject.transaction(requires_new: true) do
-            raise ActiveRecord::Rollback if catch(:abort_transaction) do
-              yield(destination, action, *transition_args); nil
-            end
-          end
-        rescue => error
-          if try_again?(max_attempts_count)
-            Retry(error)
-          else
-            Exception(error)
-          end
-          raise
-        ensure
-          if status.succeeded == false
-            status.action_pool.each do |reversable_action|
-              next unless reversable_action.respond_to?(:undo)
-              reversable_action.undo(self, *transition_args)
-            end
-          end
-        end
-      end
-    end
   end
 end

data/lib/dirty_pipeline/status.rb

@@ -16,26 +16,18 @@ module DirtyPipeline
     end
 
     def success?
-      succeeded
+      !!succeeded
     end
 
     def when_success(callback = nil)
       return self unless success?
-      if block_given?
-        yield(self)
-      else
-        callback.call(self)
-      end
+      block_given? ? yield(self) : callback.(self)
       self
     end
 
     def when_failed(callback = nil)
       return self unless storage.failed?
-      if block_given?
-        yield(self)
-      else
-        callback.call(self)
-      end
+      block_given? ? yield(self) : callback.(self)
       self
     end
 
@@ -46,11 +38,7 @@ module DirtyPipeline
 
     def when_error(callback = nil)
       return self unless errored?
-      if block_given?
-        yield(self)
-      else
-        callback.call(self)
-      end
+      block_given? ? yield(self) : callback.(self)
       self
     end
 
@@ -60,11 +48,7 @@ module DirtyPipeline
 
     def when_processing(callback = nil)
       return self unless storage.processing?
-      if block_given?
-        yield(self)
-      else
-        callback.call(self)
-      end
+      block_given? ? yield(self) : callback.(self)
       self
     end
   end

data/lib/dirty_pipeline/storage.rb

@@ -17,19 +17,26 @@ module DirtyPipeline
     def init_store(store_field)
       self.store = subject.send(store_field).to_h
       clear! if store.empty?
-      return if (store.keys & %w(cache status events errors state)).size == 5
+      return if valid_store?
       raise InvalidPipelineStorage, store
     end
 
+    def valid_store?
+      (
+        store.keys &
+          %w(status pipeline_status events errors state transaction_depth)
+      ).size == 6
+    end
+
     def clear
       self.store = subject.send(
         "#{field}=",
         "status" => nil,
         "pipeline_status" => nil,
         "state" => {},
-        "cache" => {},
         "events" => [],
         "errors" => [],
+        "transaction_depth" => 1
       )
       DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
     end
@@ -43,7 +50,8 @@ module DirtyPipeline
       events << {
         "transition" => transition,
         "args" => args,
-        "created_at" => Time.now
+        "created_at" => Time.now,
+        "cache" => {},
       }
       increment_attempts_count
       self.pipeline_status = PROCESSING_STATUS
@@ -100,7 +108,7 @@ module DirtyPipeline
 
     def commit_pipeline_status!(value = nil)
       self.pipeline_status = value
-      store["cache"].clear
+      last_event["cache"].clear
       commit!
     end
     alias :reset_pipeline_status! :commit_pipeline_status!
@@ -138,6 +146,28 @@ module DirtyPipeline
       errors.last.to_h
     end
 
+    def reset_transaction_depth
+      store["transaction_depth"] = 1
+    end
+
+    def reset_transaction_depth!
+      reset_transaction_depth
+      commit!
+    end
+
+    def transaction_depth
+      store["transaction_depth"]
+    end
+
+    def increment_transaction_depth
+      store["transaction_depth"] = store["transaction_depth"].to_i + 1
+    end
+
+    def increment_transaction_depth!
+      increment_transaction_depth
+      commit!
+    end
+
     def increment_attempts_count
       last_event.merge!(
         "attempts_count" => last_event["attempts_count"].to_i + 1

data/lib/dirty_pipeline/transaction.rb

@@ -0,0 +1,76 @@
+module DirtyPipeline
+  class Transaction
+    attr_reader :locker, :storage, :subject, :pipeline
+    def initialize(pipeline)
+      @pipeline = pipeline
+      @storage = pipeline.storage
+      @subject = pipeline.subject
+      @locker = Locker.new(@subject, @storage)
+    end
+
+    def call(*args)
+      locker.with_lock(*args) do |transition, *transition_args|
+        pipeline.schedule_cleanup
+        begin
+          destination, action, max_attempts_count =
+            find_transition(transition).values_at(:to, :action, :attempts)
+
+          # status.action_pool.unshift(action)
+          subject.transaction(requires_new: true) do
+            raise ActiveRecord::Rollback if catch(:abort_transaction) do
+              yield(destination, action, *transition_args); nil
+            end
+          end
+        rescue => error
+          if try_again?(max_attempts_count)
+            Retry(error)
+          else
+            Exception(error)
+          end
+          raise
+        ensure
+          unless pipeline.status.success?
+            storage.events
+                   .last(storage.transaction_depth)
+                   .reverse
+                   .each do |params|
+              transition = params["transition"]
+              targs = params["args"]
+              reversable_action = find_transition(transition).fetch(:action)
+              reversable_action.undo(self, *targs)
+            end
+          end
+        end
+      end
+    end
+
+    private
+
+    def Retry(error, *args)
+      storage.save_retry!(error)
+      pipeline.schedule_retry
+    end
+
+    def Exception(error)
+      storage.save_exception!(error)
+      pipeline.status.error = error
+      pipeline.status.succeeded = false
+    end
+
+    def try_again?(max_attempts_count)
+      return false unless max_attempts_count
+      storage.last_event["attempts_count"].to_i < max_attempts_count
+    end
+
+    def find_transition(name)
+      if (const_name = pipeline.class.const_get(name) rescue nil)
+        name = const_name.to_s
+      end
+      pipeline.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
+        next if from == Array(storage.status)
+        next if from.include?(storage.status.to_s)
+        raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/version.rb

@@ -1,3 +1,3 @@
 module DirtyPipeline
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/dirty_pipeline/worker.rb

@@ -0,0 +1,20 @@
+module DirtyPipeline
+  class Worker
+    include Sidekiq::Worker
+
+    sidekiq_options queue: "default",
+                    retry: 1,
+                    dead: true
+
+    # args should contain - "enqueued_pipelines" - Array of Pipeline children
+    # args should contain - some args to find_subject
+    # args should contain - some args to make transition
+
+    def perform(options)
+      # FIXME: get rid of ActiveSupport #constantize
+      pipeline_klass = options.fetch("enqueued_pipeline").constantize
+      subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
+      pipeline_klass.new(subject).call(*options.fetch("transition_args"))
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dirty_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Sergey Dolganov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-24 00:00:00.000000000 Z
+date: 2018-08-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -75,8 +75,10 @@ files:
 - lib/dirty_pipeline/locker.rb
 - lib/dirty_pipeline/status.rb
 - lib/dirty_pipeline/storage.rb
+- lib/dirty_pipeline/transaction.rb
 - lib/dirty_pipeline/transition.rb
 - lib/dirty_pipeline/version.rb
+- lib/dirty_pipeline/worker.rb
 homepage: https://github.com/sclinede/dirty_pipeline
 licenses:
 - MIT