dirty_pipeline 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c98084e4a9f05e60dbfb593de8661371a475d63d8831e80873eb0721b9b5fa6
-  data.tar.gz: 445c7d82b17abac6ab4d9c795e1e715bd11ed3bcb944a42e9897ae9c95ef06b9
+  metadata.gz: 2d7bb36236434b64e26818af5a9ba0aa299035304fbbedc2dc305111b93b58a2
+  data.tar.gz: efed49a5b15c73610497df6ca1d6e05018efa586fbd8ae3d11df5f6092414fe6
 SHA512:
-  metadata.gz: d1cec2be049b6a10eba0629d04287a685ffa6f00af52333647bc342dad67367a3de662b29d6db6e491b51dcbadf75f3d6b118fcf2bf3b5bf601c102fe522d051
-  data.tar.gz: 90ac27df1e8166cc687f76e299350e0a069b983d72c039acd0a4d33881b2b1f9cc728bc0115f3a9af95a0cc781cfba94265838d55ebb8510a90f64e1a4252e0f
+  metadata.gz: c4324152fcc2d3e403ae93e23916fa2745820d199ed277cdbeef6d93b5d8dc726235a368de1c6e95fe6b124442c32e0b7e98fd531b74baef29b197c8f2bf702d
+  data.tar.gz: ec4cf7fbf3e02ec46bd89381f7534bde8a023cfa07f5cf0c3bfc5f3b040be49e7e4457ecfec6677fd01e7f2832b3012b7a824cf199acc94db5f6cb323d1fab13

data/lib/dirty_pipeline.rb

@@ -1,13 +1,16 @@
 require "dirty_pipeline/version"
 
 module DirtyPipeline
-  require_relative "dirty_pipeline/locker.rb"
-  require_relative "dirty_pipeline/storage.rb"
-  require_relative "dirty_pipeline/transition.rb"
+  require_relative "dirty_pipeline/ext/camelcase.rb"
   require_relative "dirty_pipeline/status.rb"
+  require_relative "dirty_pipeline/storage.rb"
   require_relative "dirty_pipeline/worker.rb"
   require_relative "dirty_pipeline/transaction.rb"
+  require_relative "dirty_pipeline/event.rb"
+  require_relative "dirty_pipeline/queue.rb"
+  require_relative "dirty_pipeline/railway.rb"
   require_relative "dirty_pipeline/base.rb"
+  require_relative "dirty_pipeline/transition.rb"
 
   # This method should yield raw Redis connection
   def self.with_redis

data/lib/dirty_pipeline/base.rb

@@ -2,14 +2,7 @@ module DirtyPipeline
   class Base
     DEFAULT_RETRY_DELAY = 5 * 60 # 5 minutes
     DEFAULT_CLEANUP_DELAY = 60 * 60 * 24 # 1 day
-    RESERVED_STATUSES = [
-      Storage::FAILED_STATUS,
-      Storage::PROCESSING_STATUS,
-      Storage::RETRY_STATUS,
-      Locker::CLEAN,
-    ]
-
-    class ReservedStatusError < StandardError; end
+
     class InvalidTransition < StandardError; end
 
     class << self
@@ -19,144 +12,153 @@ module DirtyPipeline
 
       attr_reader :transitions_map
       def inherited(child)
-        child.instance_variable_set(:@transitions_map, Hash.new)
+        child.instance_variable_set(
+          :@transitions_map,
+          transitions_map || Hash.new
+        )
       end
-      # PG JSONB column
-      # {
-      #   status: :errored,
-      #   state: {
-      #     field: "value",
-      #   },
-      #   errors: [
-      #     {
-      #       error: "RuPost::API::Error",
-      #       error_message: "Timeout error",
-      #       created_at: 2018-01-01T13:22Z
-      #     },
-      #   ],
-      #   events: [
-      #     {
-      #       action: Init,
-      #       input: ...,
-      #       created_at: ...,
-      #       updated_at: ...,
-      #       attempts_count: 2,
-      #     },
-      #     {...},
-      #   ]
-      # }
       attr_accessor :pipeline_storage, :retry_delay, :cleanup_delay
 
-      def transition(action, from:, to:, name: action.to_s, attempts: 1)
-        raise ReservedStatusError unless valid_statuses?(from, to)
-        @transitions_map[name] = {
+      using StringCamelcase
+
+      def transition(name, from:, to:, action: nil, attempts: 1)
+        action ||= const_get(name.to_s.camelcase)
+        @transitions_map[name.to_s] = {
           action: action,
           from: Array(from).map(&:to_s),
           to: to.to_s,
           attempts: attempts,
         }
       end
-
-      private
-
-      def valid_statuses?(from, to)
-        ((Array(to) + Array(from)) & RESERVED_STATUSES).empty?
-      end
     end
 
-    attr_reader :subject, :error, :storage, :status, :transitions_chain
-    def initialize(subject)
+    attr_reader :subject, :storage, :status, :uuid, :queue, :railway
+    def initialize(subject, uuid: nil)
+      @uuid = uuid || Nanoid.generate
       @subject = subject
       @storage = Storage.new(subject, self.class.pipeline_storage)
-      @status = Status.new(self)
-      @transitions_chain = []
+      @railway = Railway.new(subject, @uuid)
+      @status = Status.success(subject)
     end
 
-    def enqueue(transition_name, *args)
-      DirtyPipeline::Worker.perform_async(
-        "enqueued_pipeline" => self.class.to_s,
-        "find_subject_args" => find_subject_args,
-        "transition_args" => args.unshift(transition_name),
-      )
+    def find_transition(name)
+      self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
+        next unless railway.operation.eql?(:call)
+        next if from == Array(storage.status)
+        next if from.include?(storage.status.to_s)
+        raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
+      end
     end
 
     def reset!
-      storage.reset_pipeline_status!
+      railway.clear!
     end
 
     def clear!
       storage.clear!
-    end
-
-    def cache
-      storage.last_event["cache"]
+      reset!
     end
 
     def chain(*args)
-      transitions_chain << args
+      railway[:call] << Event.create(*args, tx_id: @uuid)
       self
     end
 
-    def execute
-      Result() do
-        transitions_chain.each do |targs|
-          call(*targs)
-          storage.increment_transaction_depth!
-        end
-        storage.reset_transaction_depth!
-        transitions_chain.clear
-      end
+    def call
+      return self if (serialized_event = railway.next).nil?
+      execute(load_event(serialized_event))
     end
+    alias :call_next :call
 
-    def call(*args)
-      storage.reset_transaction_depth! if transitions_chain.empty?
-      Result() do
-        after_commit = nil
-        # transaction with support of external calls
-        transaction(*args) do |destination, action, *targs|
-          output = {}
-          fail_cause = nil
-
-          output, *after_commit = catch(:success) do
-            fail_cause = catch(:fail_with_error) do
-              Abort() if catch(:abort) do
-                throw :success, action.(self, *targs)
-              end
-            end
-            nil
-          end
+    def clean
+      railway.switch_to(:undo)
+      call_next
+      self
+    end
 
-          if fail_cause
-            Failure(fail_cause)
-          else
-            Success(destination, output)
-          end
-        end
+    def retry
+      return unless (event = load_event(railway.queue.processing_event))
+      execute(event, :retry)
+    end
 
-        Array(after_commit).each { |cb| cb.call(subject) } if after_commit
-      end
+    def schedule_cleanup
+      schedule("cleanup", cleanup_delay)
     end
 
     def schedule_retry
-      ::DirtyPipeline::Worker.perform_in(
-        retry_delay,
-        "enqueued_pipeline" => self.class.to_s,
-        "find_subject_args" => find_subject_args,
-        "retry" => true,
-      )
+      schedule("retry", retry_delay)
     end
 
-    def schedule_cleanup
-      ::DirtyPipeline::Worker.perform_in(
-        cleanup_delay,
+    def schedule(operation, delay = nil)
+      job_args = {
+        "transaction_id" => @uuid,
         "enqueued_pipeline" => self.class.to_s,
         "find_subject_args" => find_subject_args,
-        "transition_args" => [Locker::CLEAN],
-      )
+        "operation" => operation,
+      }
+
+      if delay.nil?
+        ::DirtyPipeline::Worker.perform_async(job_args)
+      else
+        ::DirtyPipeline::Worker.perform_in(delay, job_args)
+      end
+    end
+
+    def when_success
+      yield(status.data, self) if status.success?
+      self
+    end
+
+    def when_failure(tag = status.tag)
+      yield(status.data, self) if status.failure? && status.tag == tag
+      self
     end
 
     private
 
+    def execute(event, type = :call)
+      transaction(event).public_send(type) do |destination, action, *args|
+        state_changes = process_action(action, event, *args)
+        next if status.failure?
+        Success(event, state_changes, destination)
+      end
+      call_next
+
+      self
+    end
+
+    def load_event(enqueued_event)
+      storage.find_event(enqueued_event.id) || enqueued_event
+    end
+
+    def process_action(action, event, *args)
+      return catch(:success) do
+        return if interupt_on_error(event) do
+          return if interupt_on_abort(event) do
+            throw :success, run_operation(action, event, *args); nil
+          end
+        end
+        nil
+      end
+    rescue => exception
+      @status = Status.failure(exception, tag: :exception)
+      raise
+    end
+
+    def run_operation(action, event, *args)
+      return unless action.respond_to?(operation = railway.operation)
+      action.public_send(operation, event, self, *args)
+    end
+
+    def interupt_on_error(event)
+      return if (fail_cause = catch(:fail_with_error) { yield; nil }).nil?
+      Failure(event, fail_cause)
+    end
+
+    def interupt_on_abort(event)
+      Abort(event) if catch(:abort) { yield; nil }
+    end
+
     def find_subject_args
       subject.id
     end
@@ -169,31 +171,27 @@ module DirtyPipeline
       self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
     end
 
-    def transaction(*args)
-      ::DirtyPipeline::Transaction.new(self).call(*args) do |*targs|
-        yield(*targs)
-      end
-    end
-
-    def Result()
-      status.wrap { yield }
+    def transaction(event)
+      ::DirtyPipeline::Transaction.new(self, railway.queue, event)
     end
 
-    def Failure(cause)
-      storage.fail_event!
-      status.error = cause
-      status.succeeded = false
+    def Failure(event, cause)
+      event.failure!
+      railway.switch_to(:undo)
+      @status = Status.failure(cause, tag: :error)
+      throw :abort_transaction, true
     end
 
-    def Abort()
-      status.succeeded = false
+    def Abort(event)
+      event.failure!
+      railway.switch_to(:undo)
+      @status = Status.failure(subject, tag: :aborted)
       throw :abort_transaction, true
     end
 
-    def Success(destination, output)
-      cache.clear
-      storage.complete!(output, destination)
-      status.succeeded = true
+    def Success(event, changes, destination)
+      event.complete(changes, destination)
+      @status = Status.success(subject)
     end
   end
 end

data/lib/dirty_pipeline/event.rb

@@ -0,0 +1,107 @@
+require 'json'
+
+module DirtyPipeline
+  class Event
+    NEW = "new".freeze
+    START = "started".freeze
+    FAILURE = "failed".freeze
+    RETRY = "retry".freeze
+    SUCCESS = "success".freeze
+
+    def self.create(transition, *args, tx_id:)
+      new(
+        data: {
+          "uuid" => Nanoid.generate,
+          "transaction_uuid" => tx_id,
+          "transition" => transition,
+          "args" => args,
+        }
+      )
+    end
+
+    def self.load(json)
+      return unless json
+      new(JSON.load(json))
+    end
+
+    def self.dump(event)
+      JSON.dump(event.to_h)
+    end
+
+    def dump
+      self.class.dump(self)
+    end
+
+    attr_reader :id, :tx_id, :error, :data
+    def initialize(options = {}, data: nil,  error: nil)
+      unless options.empty?
+        options_hash = options.to_h
+        data  ||= options_hash["data"]
+        error ||= options_hash["error"]
+        transition = options_hash["transition"]
+        args       = options_hash["args"]
+      end
+
+      data_hash = data.to_h
+
+      @tx_id = data_hash.fetch("transaction_uuid")
+      @id = data_hash.fetch("uuid")
+      @data = {
+        "uuid" => @id,
+        "transaction_uuid" => @tx_id,
+        "transition" => transition,
+        "args" => args,
+        "created_at" => Time.now,
+        "cache" => {},
+        "attempts_count" => 1,
+        "status" => NEW,
+      }.merge(data_hash)
+      @error = error
+    end
+
+    def to_h
+      {data: @data, error: @error}
+    end
+
+    %w(args transition cache destination changes).each do |method_name|
+      define_method("#{method_name}") { @data[method_name] }
+    end
+
+    %w(new start retry failure).each do |method_name|
+      define_method("#{method_name}?") do
+        @data["status"] == self.class.const_get(method_name.upcase)
+      end
+
+      define_method("#{method_name}!") do
+        @data["status"] = self.class.const_get(method_name.upcase)
+      end
+    end
+
+    def link_exception(exception)
+      @error = {
+        "exception" => exception.class.to_s,
+        "exception_message" => exception.message,
+        "created_at" => Time.current,
+      }
+      failure!
+    end
+
+    def attempts_count
+      @data["attempts_count"].to_i
+    end
+
+    def attempt_retry
+      @data["updated_at"] = Time.now
+      @data["attempts_count"] = attempts_count + 1
+    end
+
+    def complete(changes, destination)
+      @data.merge!(
+        "destination" => destination,
+        "changes" => changes,
+        "updated_at" => Time.now,
+        "status" => SUCCESS,
+      )
+    end
+  end
+end

data/lib/dirty_pipeline/ext/camelcase.rb

@@ -0,0 +1,63 @@
+module DirtyPipeline
+  module StringCamelcase
+    refine String do
+      # rubocop:disable Metrics/LineLength
+      # See gem Facets String#camelcase
+      #
+      # Converts a string to camelcase. This method leaves the first character
+      # as given. This allows other methods to be used first, such as #uppercase
+      # and #lowercase.
+      #
+      #   "camel_case".camelcase          #=> "camelCase"
+      #   "Camel_case".camelcase          #=> "CamelCase"
+      #
+      # Custom +separators+ can be used to specify the patterns used to determine
+      # where capitalization should occur. By default these are underscores (`_`)
+      # and space characters (`\s`).
+      #
+      #   "camel/case".camelcase('/')     #=> "camelCase"
+      #
+      # If the first separator is a symbol, either `:lower` or `:upper`, then
+      # the first characters of the string will be downcased or upcased respectively.
+      #
+      #   "camel_case".camelcase(:upper)  #=> "CamelCase"
+      #
+      # Note that this implementation is different from ActiveSupport's.
+      # If that is what you are looking for you may want {#modulize}.
+      #
+      # rubocop:enable Metrics/LineLength
+
+      # rubocop:disable Metrics/AbcSize,Metrics/MethodLength
+      def camelcase(*separators)
+        case separators.first
+        when Symbol, TrueClass, FalseClass, NilClass
+          first_letter = separators.shift
+        end
+
+        separators = ["_", '\s'] if separators.empty?
+
+        str = dup
+
+        separators.each do |s|
+          str = str.gsub(/(?:#{s}+)([a-z])/) do
+            Regexp.last_match(1).upcase
+          end
+        end
+
+        case first_letter
+        when :upper, true
+          str = str.gsub(/(\A|\s)([a-z])/) do
+            Regexp.last_match(1) + Regexp.last_match(2).upcase
+          end
+        when :lower, false
+          str = str.gsub(/(\A|\s)([A-Z])/) do
+            Regexp.last_match(1) + Regexp.last_match(2).downcase
+          end
+        end
+
+        str
+      end
+      # rubocop:enable Metrics/AbcSize,Metrics/MethodLength
+    end
+  end
+end

data/lib/dirty_pipeline/queue.rb

@@ -0,0 +1,82 @@
+module DirtyPipeline
+  class Queue
+    attr_reader :root
+    def initialize(operation, subject, transaction_id)
+      @root = "dirty-pipeline-queue:#{subject.class}:#{subject.id}:" \
+              "op_#{operation}:txid_#{transaction_id}"
+    end
+
+    def clear!
+      DirtyPipeline.with_redis do |r|
+        r.del active_event_key
+        r.del events_queue_key
+      end
+    end
+
+    def to_a
+      DirtyPipeline.with_redis { |r| r.lrange(events_queue_key, 0, -1) }
+    end
+
+    def push(event)
+      DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
+    end
+    alias :<< :push
+
+    def unshift(event)
+      DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
+    end
+
+    def dequeue
+      DirtyPipeline.with_redis do |r|
+        data = r.lpop(events_queue_key)
+        data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
+        return unpack(data)
+      end
+    end
+    alias :pop :dequeue
+
+    def event_in_progress?(event = nil)
+      if event.nil?
+        !processing_event.nil?
+      else
+        processing_event.id == event.id
+      end
+    end
+
+    def processing_event
+      DirtyPipeline.with_redis { |r| unpack r.get(active_event_key) }
+    end
+
+    private
+
+    def pack(event)
+      JSON.dump(
+        "evid" => event.id,
+        "txid" => event.tx_id,
+        "transit" => event.transition,
+        "args" => event.args,
+      )
+    end
+
+    def unpack(packed_event)
+      return unless packed_event
+      unpacked_event = JSON.load(packed_event)
+      Event.new(
+        data: {
+          "uuid" => unpacked_event["evid"],
+          "transaction_uuid" => unpacked_event["txid"],
+          "transition" => unpacked_event["transit"],
+          "args" => unpacked_event["args"],
+        }
+      )
+    end
+
+    def events_queue_key
+      "#{root}:events"
+    end
+
+    def active_event_key
+      "#{root}:active"
+    end
+  end
+end

data/lib/dirty_pipeline/railway.rb

@@ -0,0 +1,74 @@
+module DirtyPipeline
+  class Railway
+    OPERATIONS = %w(call undo finalize)
+
+    def initialize(subject, transaction_id)
+      @tx_id = transaction_id
+      @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:" \
+              ":txid_#{transaction_id}"
+      @queues = Hash[
+        OPERATIONS.map do |operation|
+          [operation, Queue.new(operation, subject, transaction_id)]
+        end
+      ]
+    end
+
+    def clear!
+      @queues.values.each(&:clear!)
+      DirtyPipeline.with_redis { |r| r.del(active_operation_key) }
+    end
+
+    def next
+      return if other_transaction_in_progress?
+      start_transaction! if running_transaction.nil?
+
+      queue.pop.tap { |event| finish_transaction! if event.nil? }
+    end
+
+    def queue(name = active)
+      @queues[name.to_s]
+    end
+    alias :[] :queue
+
+    def switch_to(name)
+      raise ArgumentError unless OPERATIONS.include?(name.to_s)
+      return if name.to_s == active
+      DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
+    end
+
+    def active
+      DirtyPipeline.with_redis { |r| r.get(active_operation_key) }
+    end
+    alias :operation :active
+
+    private
+
+    def active_transaction_key
+      "#{@root}:active_transaction"
+    end
+
+    def active_operation_key
+      "#{@root}:active_operation"
+    end
+
+    def start_transaction!
+      switch_to(OPERATIONS.first)
+      DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
+    end
+
+    def finish_transaction!
+      return unless running_transaction == @tx_id
+      DirtyPipeline.with_redis { |r| r.del(active_transaction_key) }
+      @queues.values.each(&:clear!)
+    end
+
+   def running_transaction
+     DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
+   end
+
+    def other_transaction_in_progress?
+      return false if running_transaction.nil?
+      running_transaction != @tx_id
+    end
+  end
+end

data/lib/dirty_pipeline/status.rb

@@ -1,55 +1,27 @@
 module DirtyPipeline
-  class Status < SimpleDelegator
-    attr_accessor :error, :succeeded, :action_pool
-    attr_reader :storage, :pipeline
-    def initialize(*)
-      super
-      @storage = __getobj__.storage
-      @action_pool = []
-    end
+  class Status
+    attr_reader :success, :tag, :data
 
-    def wrap
-      return self if succeeded == false
-      self.succeeded = nil
-      yield
-      self
+    def self.success(data, tag: :success)
+      new(true, data, tag)
     end
 
-    def success?
-      !!succeeded
+    def self.failure(data, tag: :exception)
+      new(false, data, tag)
     end
 
-    def when_success(callback = nil)
-      return self unless success?
-      block_given? ? yield(self) : callback.(self)
-      self
+    def initialize(success, data, tag = nil)
+      @success = success
+      @data = data
+      @tag = tag
     end
 
-    def when_failed(callback = nil)
-      return self unless storage.failed?
-      block_given? ? yield(self) : callback.(self)
-      self
-    end
-
-    def errored?
-      return if succeeded.nil?
-      ready? && !succeeded
-    end
-
-    def when_error(callback = nil)
-      return self unless errored?
-      block_given? ? yield(self) : callback.(self)
-      self
-    end
-
-    def ready?
-      storage.pipeline_status.nil?
+    def success?
+      !!success
     end
 
-    def when_processing(callback = nil)
-      return self unless storage.processing?
-      block_given? ? yield(self) : callback.(self)
-      self
+    def failure?
+      !success?
     end
   end
 end

data/lib/dirty_pipeline/storage.rb

@@ -1,11 +1,12 @@
 module DirtyPipeline
   class Storage
-    FAILED_STATUS = "failed".freeze
+    SUCCESS_STATUS = "success".freeze
+    FAILURE_STATUS = "failure".freeze
     RETRY_STATUS = "retry".freeze
     PROCESSING_STATUS = "processing".freeze
     class InvalidPipelineStorage < StandardError; end
 
-    attr_reader :subject, :field
+    attr_reader :subject, :field, :transactions_queue
     attr_accessor :store
     alias :to_h :store
     def initialize(subject, field)
@@ -16,210 +17,75 @@ module DirtyPipeline
 
     def init_store(store_field)
       self.store = subject.send(store_field).to_h
-      clear! if store.empty?
+      clear if store.empty?
       return if valid_store?
       raise InvalidPipelineStorage, store
     end
 
     def valid_store?
-      (
-        store.keys &
-          %w(status pipeline_status events errors state transaction_depth)
-      ).size == 6
-    end
-
+      (store.keys & %w(status events errors state)).size.eql?(4)
+    end
+
+    # PG JSONB column
+    # {
+    #   status: :errored,
+    #   state: {
+    #     field: "value",
+    #   },
+    #   errors: {
+    #     "<event_id>": {
+    #       error: "RuPost::API::Error",
+    #       error_message: "Timeout error",
+    #       created_at: 2018-01-01T13:22Z
+    #     },
+    #   },
+    #   events: {
+    #     <event_id>: {
+    #       action: Init,
+    #       input: ...,
+    #       created_at: ...,
+    #       updated_at: ...,
+    #       attempts_count: 2,
+    #     },
+    #     <event_id>: {...},
+    #   }
+    # }
     def clear
       self.store = subject.send(
         "#{field}=",
         "status" => nil,
-        "pipeline_status" => nil,
         "state" => {},
-        "events" => [],
-        "errors" => [],
-        "transaction_depth" => 1
+        "events" => {},
+        "errors" => {}
       )
-      DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
     end
 
     def clear!
       clear
-      commit!
-    end
-
-    def start!(transition, args)
-      events << {
-        "transition" => transition,
-        "args" => args,
-        "created_at" => Time.now,
-        "cache" => {},
-      }
-      increment_attempts_count
-      self.pipeline_status = PROCESSING_STATUS
-      # self.status = "processing", should be set by Locker
-      commit!
-    end
-
-    def start_retry!
-      last_event.merge!(updated_at: Time.now)
-      increment_attempts_count
-      self.pipeline_status = PROCESSING_STATUS
-      # self.status = "processing", should be set by Locker
-      commit!
-    end
-
-    def complete!(output, destination)
-      store["status"] = destination
-      state.merge!(output)
-      last_event.merge!(
-        "output" => output,
-        "updated_at" => Time.now,
-        "success" => true,
-      )
-      commit!
-    end
-
-    def fail_event!
-      fail_event
-      commit!
-    end
-
-    def fail_event
-      last_event["failed"] = true
+      subject.update_attributes!(field => store)
     end
 
     def status
       store["status"]
     end
 
-    def pipeline_status_key
-      "pipeline-status:#{subject.class}:#{subject.id}:#{field}"
-    end
-
-    def pipeline_status=(value)
-      DirtyPipeline.with_redis do |r|
-        if value
-          r.set(pipeline_status_key, value)
-        else
-          r.del(pipeline_status_key)
-        end
-      end
-      store["pipeline_status"] = value
-    end
-
-    def commit_pipeline_status!(value = nil)
-      self.pipeline_status = value
-      last_event["cache"].clear
-      commit!
-    end
-    alias :reset_pipeline_status! :commit_pipeline_status!
-
-    def pipeline_status
-      DirtyPipeline.with_redis do |r|
-        store["pipeline_status"] = r.get(pipeline_status_key)
-      end
-      store.fetch("pipeline_status")
-    end
-
-    def state
-      store.fetch("state")
-    end
-
-    def events
-      store.fetch("events")
-    end
-
-    def last_event
-      events.last.to_h
-    end
-
-    def last_event_error(event_idx = nil)
-      event = events[event_idx] if event_idx
-      event ||= last_event
-      errors[event["error_idx"]].to_h
-    end
-
-    def errors
-      store.fetch("errors")
-    end
-
-    def last_error
-      errors.last.to_h
-    end
-
-    def reset_transaction_depth
-      store["transaction_depth"] = 1
-    end
-
-    def reset_transaction_depth!
-      reset_transaction_depth
-      commit!
-    end
-
-    def transaction_depth
-      store["transaction_depth"]
-    end
-
-    def increment_transaction_depth
-      store["transaction_depth"] = store["transaction_depth"].to_i + 1
-    end
-
-    def increment_transaction_depth!
-      increment_transaction_depth
-      commit!
-    end
-
-    def increment_attempts_count
-      last_event.merge!(
-        "attempts_count" => last_event["attempts_count"].to_i + 1
-      )
-    end
-
-    def increment_attempts_count!
-      increment_attempts_count
-      commit!
-    end
-
-    def save_retry(error)
-      save_error(error)
-      self.pipeline_status = RETRY_STATUS
-    end
-
-    def save_retry!(error)
-      save_retry(error)
-      commit!
-    end
-
-    def save_exception(exception)
-      errors << {
-        "error" => exception.class.to_s,
-        "error_message" => exception.message,
-        "created_at" => Time.current,
-      }
-      last_event["error_idx"] = errors.size - 1
-      fail_event
-      self.pipeline_status = FAILED_STATUS
-    end
-
-    def save_exception!(error)
-      save_exception(error)
-      commit!
-    end
-
-    def commit!
+    def commit!(event)
+      store["status"] = event.destination if event.destination
+      require'pry';binding.pry unless event.changes.respond_to?(:to_h)
+      store["state"].merge!(event.changes) unless event.changes.to_h.empty?
+      store["errors"][event.id] = event.error unless event.error.to_h.empty?
+      store["events"][event.id] = event.data unless event.data.to_h.empty?
       subject.assign_attributes(field => store)
       subject.save!
     end
 
-    def ready?
-      storage.pipeline_status.nil?
-    end
-
-    def failed?
-      pipeline_status == FAILED_STATUS
+    def processing_event
+      find_event(transactions_queue.processing_event.id)
     end
 
-    def processing?
-      [PROCESSING_STATUS, RETRY_STATUS].include?(pipeline_status)
+    def find_event(event_id)
+      return unless (found_event = store["events"][event_id])
+      Event.new(data: found_event, error: store["errors"][event_id])
     end
   end
 end

data/lib/dirty_pipeline/transaction.rb

@@ -1,76 +1,56 @@
 module DirtyPipeline
   class Transaction
-    attr_reader :locker, :storage, :subject, :pipeline
-    def initialize(pipeline)
+    attr_reader :locker, :storage, :subject, :pipeline, :queue, :event
+    def initialize(pipeline, queue, event)
       @pipeline = pipeline
-      @storage = pipeline.storage
       @subject = pipeline.subject
-      @locker = Locker.new(@subject, @storage)
+      @storage = pipeline.storage
+      @queue = queue
+      @event = event
     end
 
-    def call(*args)
-      locker.with_lock(*args) do |transition, *transition_args|
-        pipeline.schedule_cleanup
-        begin
-          destination, action, max_attempts_count =
-            find_transition(transition).values_at(:to, :action, :attempts)
+    def retry
+      event.attempt_retry!
+      pipeline.schedule_cleanup
 
-          # status.action_pool.unshift(action)
-          subject.transaction(requires_new: true) do
-            raise ActiveRecord::Rollback if catch(:abort_transaction) do
-              yield(destination, action, *transition_args); nil
-            end
-          end
-        rescue => error
-          if try_again?(max_attempts_count)
-            Retry(error)
-          else
-            Exception(error)
-          end
-          raise
-        ensure
-          unless pipeline.status.success?
-            storage.events
-                   .last(storage.transaction_depth)
-                   .reverse
-                   .each do |params|
-              transition = params["transition"]
-              targs = params["args"]
-              reversable_action = find_transition(transition).fetch(:action)
-              reversable_action.undo(self, *targs)
-            end
-          end
-        end
-      end
+      with_transaction { |*targs| yield(*targs) }
     end
 
-    private
+    def call
+      # return unless queue.event_in_progress?(event)
 
-    def Retry(error, *args)
-      storage.save_retry!(error)
-      pipeline.schedule_retry
-    end
+      event.start!
+      pipeline.schedule_cleanup
 
-    def Exception(error)
-      storage.save_exception!(error)
-      pipeline.status.error = error
-      pipeline.status.succeeded = false
+      with_transaction { |*targs| yield(*targs) }
     end
 
-    def try_again?(max_attempts_count)
-      return false unless max_attempts_count
-      storage.last_event["attempts_count"].to_i < max_attempts_count
-    end
+    private
+
+    def with_transaction
+      destination, action, max_attempts_count =
+        pipeline.find_transition(event.transition)
+                .values_at(:to, :action, :attempts)
 
-    def find_transition(name)
-      if (const_name = pipeline.class.const_get(name) rescue nil)
-        name = const_name.to_s
+      storage.commit!(event)
+
+      # status.action_pool.unshift(action)
+      subject.transaction(requires_new: true) do
+        raise ActiveRecord::Rollback if catch(:abort_transaction) do
+          yield(destination, action, *event.args); nil
+        end
       end
-      pipeline.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
-        next if from == Array(storage.status)
-        next if from.include?(storage.status.to_s)
-        raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
+    rescue => exception
+      event.link_exception(exception)
+      if max_attempts_count.to_i > event.attempts_count
+        event.retry!
+        pipeline.schedule_retry
+      else
+        pipeline.schedule_cleanup
       end
+      raise
+    ensure
+      storage.commit!(event)
     end
   end
 end

data/lib/dirty_pipeline/transition.rb

@@ -8,32 +8,41 @@ module DirtyPipeline
       throw :fail_with_error, error
     end
 
-    def Success(output = nil, after_commit: nil, &block)
-      result = [output.to_h]
-      after_commit = Array(after_commit) << block if block_given?
-      result += Array(after_commit) if after_commit
-      throw :success, result
+    def Success(changes = nil)
+      throw :success, changes.to_h
+    end
+
+    def self.finalize(*args, **kwargs)
+      event, pipeline, *args = args
+      instance = new(event, *args, **kwargs)
+      pipeline.railway.switch_to(:call)
+      return unless instance.respond_to?(:finalize)
+      instance.finalize(pipeline.subject)
     end
 
     def self.undo(*args, **kwargs)
-      pipeline = args.shift
-      instance = new(pipeline, *args, **kwargs)
+      event, pipeline, *args = args
+      instance = new(event, *args, **kwargs)
       return unless instance.respond_to?(:undo)
       instance.undo(pipeline.subject)
     end
 
     def self.call(*args, **kwargs)
-      pipeline = args.shift
-      new(pipeline, *args, **kwargs).call(pipeline.subject)
+      event, pipeline, *args = args
+      instance = new(event, *args, **kwargs)
+      pipeline.railway[:undo] << event if instance.respond_to?(:undo)
+      pipeline.railway[:finalize] << event if instance.respond_to?(:finalize)
+      pipeline.railway.switch_to(:finalize) if instance.respond_to?(:finalize)
+      new(event, *args, **kwargs).call(pipeline.subject)
     end
 
-    attr_reader :pipeline
-    def initialize(pipeline, *, **)
-      @pipeline = pipeline
+    attr_reader :event
+    def initialize(event, *, **)
+      @event = event
     end
 
     def fetch(key)
-      pipeline.cache.fetch(key) { pipeline.cache[key] = yield }
+      event.cache.fetch(key) { event.cache[key] = yield }
     end
   end
 end

data/lib/dirty_pipeline/version.rb

@@ -1,3 +1,3 @@
 module DirtyPipeline
-  VERSION = "0.4.0"
+  VERSION = "0.5.0"
 end

data/lib/dirty_pipeline/worker.rb

@@ -1,20 +1,27 @@
+require 'sidekiq'
 module DirtyPipeline
   class Worker
     include Sidekiq::Worker
+    using StringCamelcase
 
-    sidekiq_options queue: "default",
-                    retry: 1,
-                    dead: true
-
-    # args should contain - "enqueued_pipelines" - Array of Pipeline children
+    # args should contain - "enqueued_pipeline"
     # args should contain - some args to find_subject
     # args should contain - some args to make transition
-
-    def perform(options)
-      # FIXME: get rid of ActiveSupport #constantize
-      pipeline_klass = options.fetch("enqueued_pipeline").constantize
+    def perform(options = {})
+      pipeline_klass =
+        Kernel.const_get(options.fetch("enqueued_pipeline").to_s.camelcase)
       subject = pipeline_klass.find_subject(*options.fetch("find_subject_args"))
-      pipeline_klass.new(subject).call(*options.fetch("transition_args"))
+      transaction_id = options.fetch("transaction_id")
+      pipeline = pipeline_klass.new(subject, uuid: transaction_id)
+      operation = options.fetch("operation")
+
+      case operation
+      when "cleanup"
+        pipeline.clean
+      when "retry"
+        return pipeline.retry
+      end
+      pipeline.call
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dirty_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Sergey Dolganov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-25 00:00:00.000000000 Z
+date: 2018-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -72,7 +72,10 @@ files:
 - dirty_pipeline.gemspec
 - lib/dirty_pipeline.rb
 - lib/dirty_pipeline/base.rb
-- lib/dirty_pipeline/locker.rb
+- lib/dirty_pipeline/event.rb
+- lib/dirty_pipeline/ext/camelcase.rb
+- lib/dirty_pipeline/queue.rb
+- lib/dirty_pipeline/railway.rb
 - lib/dirty_pipeline/status.rb
 - lib/dirty_pipeline/storage.rb
 - lib/dirty_pipeline/transaction.rb

data/lib/dirty_pipeline/locker.rb

@@ -1,108 +0,0 @@
-module DirtyPipeline
-  class Locker
-    CLEAN = "clean".freeze
-
-    attr_reader :storage, :subject
-    def initialize(subject, storage)
-      @subject = subject
-      @storage = storage
-    end
-
-    class Normal
-      attr_reader :locker, :transition, :transition_args
-      def initialize(locker, args)
-        @locker = locker
-        @transition = args.shift
-        @transition_args = args
-      end
-
-      # NORMAL MODE
-      # if state is PROCESSING_STATE - finish
-      # if state is FAILED_STATE - finish
-      # otherwise - start
-      def skip_any_action?
-        [
-          Storage::PROCESSING_STATUS,
-          Storage::FAILED_STATUS,
-        ].include?(locker.storage.pipeline_status)
-      end
-
-      def start!
-        locker.storage.start!(transition, transition_args)
-      end
-
-      def lock!
-        return if skip_any_action?
-        start!
-        begin
-          yield(transition, *transition_args)
-        ensure
-          if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
-            locker.storage.reset_pipeline_status!
-          end
-        end
-      rescue
-        if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
-          locker.storage.commit_pipeline_status!(Storage::FAILED_STATUS)
-        end
-        raise
-      end
-    end
-
-    class Retry < Normal
-      def initialize(locker, _args)
-        @locker = locker
-        @transition = locker.storage.last_event["transition"]
-        @transition_args = locker.storage.last_event["input"]
-      end
-
-      def skip_any_action?
-        storage.status != Storage::RETRY_STATUS
-      end
-
-      def start!
-        locker.storage.start_retry!
-      end
-    end
-
-    # run in time much more then time to process an item
-    class Clean < Retry
-      ONE_DAY = 60 * 60 * 24
-
-      def skip_any_action?
-        return true if storage.status != Storage::PROCESSING_STATUS
-        started_less_then_a_day_ago?
-      end
-
-      def started_less_then_a_day_ago?
-        return unless (updated_at = locker.storage.last_event["updated_at"])
-        updated_at > one_day_ago
-      end
-
-      def one_day_ago
-        Time.now - ONE_DAY
-      end
-    end
-
-    def with_lock(*args)
-      lock!(*args) do |transition, *transition_args|
-        yield(transition, *transition_args)
-      end
-    end
-
-    def lock!(*args)
-      locker_klass(*args).new(self, args).lock! { |*largs| yield(*largs) }
-    end
-
-    def locker_klass(transition, *)
-      case transition
-      when Storage::RETRY_STATUS
-        Retry
-      when CLEAN
-        Clean
-      else
-        Normal
-      end
-    end
-  end
-end