dirty_pipeline 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d7bb36236434b64e26818af5a9ba0aa299035304fbbedc2dc305111b93b58a2
-  data.tar.gz: efed49a5b15c73610497df6ca1d6e05018efa586fbd8ae3d11df5f6092414fe6
+  metadata.gz: e4eef96b779d46b3c749e665cf467f69577b8b1dc1bac3dd1e107be269619a3d
+  data.tar.gz: 0a42107025850c20d9d2f4bfbeb5fb4b3fd42610c9eae3aacc3ba80e3600ad6d
 SHA512:
-  metadata.gz: c4324152fcc2d3e403ae93e23916fa2745820d199ed277cdbeef6d93b5d8dc726235a368de1c6e95fe6b124442c32e0b7e98fd531b74baef29b197c8f2bf702d
-  data.tar.gz: ec4cf7fbf3e02ec46bd89381f7534bde8a023cfa07f5cf0c3bfc5f3b040be49e7e4457ecfec6677fd01e7f2832b3012b7a824cf199acc94db5f6cb323d1fab13
+  metadata.gz: 9fcb6bcf6655ab497f97fcf0f72d54a46b4a12407d5d2567d6027442e2652d4c901f0ef1a5031de184c0714821c27f466511c0db803e072082bd0d121968cbf3
+  data.tar.gz: '085965b95df37be6654977fb820b3d8937abd2097bc01a340119a8e99170206ce3670ffdcc2698723af7a484403fe76359b9b0e55548d6d11f1e2573c2ade88b'

data/.gitignore

@@ -9,3 +9,4 @@
 
 # rspec failure tracking
 .rspec_status
+Gemfile.lock

data/dirty_pipeline.gemspec

@@ -21,7 +21,15 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  # temporary dependency
+  spec.add_runtime_dependency "sidekiq", "~> 5.0"
+  spec.add_runtime_dependency "redis", "~> 4.0"
+  spec.add_runtime_dependency "nanoid", "~> 0.2.0"
+
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "dotenv", "~> 2.2"
+  spec.add_development_dependency "timecop", "~> 0.9"
+  spec.add_development_dependency "pry", "~> 0.11"
 end

data/lib/dirty_pipeline.rb

@@ -1,4 +1,5 @@
 require "dirty_pipeline/version"
+require "nanoid"
 
 module DirtyPipeline
   require_relative "dirty_pipeline/ext/camelcase.rb"

data/lib/dirty_pipeline/base.rb

@@ -22,7 +22,8 @@ module DirtyPipeline
       using StringCamelcase
 
       def transition(name, from:, to:, action: nil, attempts: 1)
-        action ||= const_get(name.to_s.camelcase)
+        action ||= const_get(name.to_s.camelcase(:upper)) rescue nil
+        action ||= method(name) if respond_to?(name)
         @transitions_map[name.to_s] = {
           action: action,
           from: Array(from).map(&:to_s),
@@ -55,7 +56,7 @@ module DirtyPipeline
     end
 
     def clear!
-      storage.clear!
+      storage.reset!
       reset!
     end
 
@@ -66,7 +67,7 @@ module DirtyPipeline
 
     def call
       return self if (serialized_event = railway.next).nil?
-      execute(load_event(serialized_event))
+      execute(load_event(serialized_event), tx_method: :call)
     end
     alias :call_next :call
 
@@ -78,7 +79,7 @@ module DirtyPipeline
 
     def retry
       return unless (event = load_event(railway.queue.processing_event))
-      execute(event, :retry)
+      execute(event, tx_id: :retry)
     end
 
     def schedule_cleanup
@@ -116,8 +117,8 @@ module DirtyPipeline
 
     private
 
-    def execute(event, type = :call)
-      transaction(event).public_send(type) do |destination, action, *args|
+    def execute(event, tx_method:)
+      transaction(event).public_send(tx_method) do |destination, action, *args|
         state_changes = process_action(action, event, *args)
         next if status.failure?
         Success(event, state_changes, destination)
@@ -134,9 +135,7 @@ module DirtyPipeline
     def process_action(action, event, *args)
       return catch(:success) do
         return if interupt_on_error(event) do
-          return if interupt_on_abort(event) do
-            throw :success, run_operation(action, event, *args); nil
-          end
+          throw :success, run_operation(action, event, *args)
         end
         nil
       end
@@ -151,14 +150,10 @@ module DirtyPipeline
     end
 
     def interupt_on_error(event)
-      return if (fail_cause = catch(:fail_with_error) { yield; nil }).nil?
+      return unless (fail_cause = catch(:fail_operation) { yield; nil })
       Failure(event, fail_cause)
     end
 
-    def interupt_on_abort(event)
-      Abort(event) if catch(:abort) { yield; nil }
-    end
-
     def find_subject_args
       subject.id
     end
@@ -172,20 +167,18 @@ module DirtyPipeline
     end
 
     def transaction(event)
-      ::DirtyPipeline::Transaction.new(self, railway.queue, event)
+      Transaction.new(self, event)
     end
 
     def Failure(event, cause)
-      event.failure!
-      railway.switch_to(:undo)
-      @status = Status.failure(cause, tag: :error)
-      throw :abort_transaction, true
-    end
-
-    def Abort(event)
-      event.failure!
       railway.switch_to(:undo)
-      @status = Status.failure(subject, tag: :aborted)
+      if cause.eql?(:abort)
+        event.abort!
+        @status = Status.failure(subject, tag: :aborted)
+      else
+        event.failure!
+        @status = Status.failure(cause, tag: :error)
+      end
       throw :abort_transaction, true
     end
 

data/lib/dirty_pipeline/event.rb

@@ -5,8 +5,9 @@ module DirtyPipeline
     NEW = "new".freeze
     START = "started".freeze
     FAILURE = "failed".freeze
+    ABORT = "aborted".freeze
     RETRY = "retry".freeze
-    SUCCESS = "success".freeze
+    SUCCESS = "succeeded".freeze
 
     def self.create(transition, *args, tx_id:)
       new(
@@ -19,33 +20,20 @@ module DirtyPipeline
       )
     end
 
-    def self.load(json)
-      return unless json
-      new(JSON.load(json))
-    end
-
-    def self.dump(event)
-      JSON.dump(event.to_h)
-    end
-
-    def dump
-      self.class.dump(self)
-    end
-
     attr_reader :id, :tx_id, :error, :data
     def initialize(options = {}, data: nil,  error: nil)
       unless options.empty?
         options_hash = options.to_h
         data  ||= options_hash["data"]
         error ||= options_hash["error"]
-        transition = options_hash["transition"]
-        args       = options_hash["args"]
       end
 
       data_hash = data.to_h
 
-      @tx_id = data_hash.fetch("transaction_uuid")
-      @id = data_hash.fetch("uuid")
+      @tx_id     = data_hash.fetch("transaction_uuid")
+      @id        = data_hash.fetch("uuid")
+      transition = data_hash.fetch("transition")
+      args       = data_hash.fetch("args").to_a
       @data = {
         "uuid" => @id,
         "transaction_uuid" => @tx_id,
@@ -67,7 +55,7 @@ module DirtyPipeline
       define_method("#{method_name}") { @data[method_name] }
     end
 
-    %w(new start retry failure).each do |method_name|
+    %w(new start retry failure success abort).each do |method_name|
       define_method("#{method_name}?") do
         @data["status"] == self.class.const_get(method_name.upcase)
       end
@@ -81,7 +69,7 @@ module DirtyPipeline
       @error = {
         "exception" => exception.class.to_s,
         "exception_message" => exception.message,
-        "created_at" => Time.current,
+        "created_at" => Time.now,
       }
       failure!
     end
@@ -90,7 +78,7 @@ module DirtyPipeline
       @data["attempts_count"].to_i
     end
 
-    def attempt_retry
+    def attempt_retry!
       @data["updated_at"] = Time.now
       @data["attempts_count"] = attempts_count + 1
     end

data/lib/dirty_pipeline/queue.rb

@@ -1,8 +1,7 @@
 module DirtyPipeline
   class Queue
-    attr_reader :root
-    def initialize(operation, subject, transaction_id)
-      @root = "dirty-pipeline-queue:#{subject.class}:#{subject.id}:" \
+    def initialize(operation, subject_class, subject_id, transaction_id)
+      @root = "dirty-pipeline-queue:#{subject_class}:#{subject_id}:" \
               "op_#{operation}:txid_#{transaction_id}"
     end
 
@@ -14,37 +13,34 @@ module DirtyPipeline
     end
 
     def to_a
-      DirtyPipeline.with_redis { |r| r.lrange(events_queue_key, 0, -1) }
+      DirtyPipeline.with_redis do |r|
+        r.lrange(events_queue_key, 0, -1).map! do |packed_event|
+          unpack(packed_event)
+        end
+      end
     end
 
     def push(event)
       DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
+      self
     end
     alias :<< :push
 
     def unshift(event)
       DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
+      self
     end
 
-    def dequeue
+    def pop
       DirtyPipeline.with_redis do |r|
         data = r.lpop(events_queue_key)
         data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
-        return unpack(data)
-      end
-    end
-    alias :pop :dequeue
-
-    def event_in_progress?(event = nil)
-      if event.nil?
-        !processing_event.nil?
-      else
-        processing_event.id == event.id
+        unpack(data)
       end
     end
 
     def processing_event
-      DirtyPipeline.with_redis { |r| unpack r.get(active_event_key) }
+      DirtyPipeline.with_redis { |r| unpack(r.get(active_event_key)) }
     end
 
     private
@@ -72,11 +68,11 @@ module DirtyPipeline
     end
 
     def events_queue_key
-      "#{root}:events"
+      "#{@root}:events"
     end
 
     def active_event_key
-      "#{root}:active"
+      "#{@root}:active"
     end
   end
 end

data/lib/dirty_pipeline/railway.rb

@@ -1,37 +1,45 @@
 module DirtyPipeline
   class Railway
-    OPERATIONS = %w(call undo finalize)
+    DEFAULT_OPERATIONS = %w(call undo finalize)
 
     def initialize(subject, transaction_id)
       @tx_id = transaction_id
-      @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:" \
-              ":txid_#{transaction_id}"
+      @subject_class = subject.class.to_s
+      @subject_id = subject.id.to_s
+      @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:"
       @queues = Hash[
-        OPERATIONS.map do |operation|
-          [operation, Queue.new(operation, subject, transaction_id)]
+        DEFAULT_OPERATIONS.map do |operation|
+          [operation, create_queue(operation)]
         end
       ]
     end
 
     def clear!
       @queues.values.each(&:clear!)
-      DirtyPipeline.with_redis { |r| r.del(active_operation_key) }
+      DirtyPipeline.with_redis do |r|
+        r.multi do |mr|
+          mr.del(active_operation_key)
+          mr.del(active_transaction_key)
+        end
+      end
     end
 
     def next
       return if other_transaction_in_progress?
-      start_transaction! if running_transaction.nil?
+      start_transaction! unless running_transaction
 
       queue.pop.tap { |event| finish_transaction! if event.nil? }
     end
 
-    def queue(name = active)
-      @queues[name.to_s]
+    def queue(operation_name = active)
+      @queues.fetch(operation_name.to_s) do
+        @queues.store(operation_name, create_queue(operation_name))
+      end
     end
     alias :[] :queue
 
     def switch_to(name)
-      raise ArgumentError unless OPERATIONS.include?(name.to_s)
+      raise ArgumentError unless DEFAULT_OPERATIONS.include?(name.to_s)
       return if name.to_s == active
       DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
     end
@@ -41,8 +49,16 @@ module DirtyPipeline
     end
     alias :operation :active
 
+    def running_transaction
+      DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
+    end
+
     private
 
+    def create_queue(operation_name)
+      Queue.new(operation_name, @subject_class, @subject_id, @tx_id)
+    end
+
     def active_transaction_key
       "#{@root}:active_transaction"
     end
@@ -52,20 +68,14 @@ module DirtyPipeline
     end
 
     def start_transaction!
-      switch_to(OPERATIONS.first)
+      switch_to(DEFAULT_OPERATIONS.first) unless active
       DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
     end
 
     def finish_transaction!
-      return unless running_transaction == @tx_id
-      DirtyPipeline.with_redis { |r| r.del(active_transaction_key) }
-      @queues.values.each(&:clear!)
+      clear! if running_transaction == @tx_id
     end
 
-   def running_transaction
-     DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
-   end
-
     def other_transaction_in_progress?
       return false if running_transaction.nil?
       running_transaction != @tx_id

data/lib/dirty_pipeline/storage.rb

@@ -1,68 +1,45 @@
 module DirtyPipeline
+  # Storage structure
+  # {
+  #   status: :errored,
+  #   state: {
+  #     field: "value",
+  #   },
+  #   errors: {
+  #     "<event_id>": {
+  #       error: "RuPost::API::Error",
+  #       error_message: "Timeout error",
+  #       created_at: 2018-01-01T13:22Z
+  #     },
+  #   },
+  #   events: {
+  #     <event_id>: {
+  #       transition: "Create",
+  #       args: ...,
+  #       changes: ...,
+  #       created_at: ...,
+  #       updated_at: ...,
+  #       attempts_count: 2,
+  #     },
+  #     <event_id>: {...},
+  #   }
+  # }
   class Storage
-    SUCCESS_STATUS = "success".freeze
-    FAILURE_STATUS = "failure".freeze
-    RETRY_STATUS = "retry".freeze
-    PROCESSING_STATUS = "processing".freeze
     class InvalidPipelineStorage < StandardError; end
 
-    attr_reader :subject, :field, :transactions_queue
-    attr_accessor :store
+    attr_reader :subject, :field, :store
     alias :to_h :store
     def initialize(subject, field)
       @subject = subject
       @field = field
-      init_store(field)
+      @store = subject.send(@field).to_h
+      reset if @store.empty?
+      raise InvalidPipelineStorage, store unless valid_store?
     end
 
-    def init_store(store_field)
-      self.store = subject.send(store_field).to_h
-      clear if store.empty?
-      return if valid_store?
-      raise InvalidPipelineStorage, store
-    end
-
-    def valid_store?
-      (store.keys & %w(status events errors state)).size.eql?(4)
-    end
-
-    # PG JSONB column
-    # {
-    #   status: :errored,
-    #   state: {
-    #     field: "value",
-    #   },
-    #   errors: {
-    #     "<event_id>": {
-    #       error: "RuPost::API::Error",
-    #       error_message: "Timeout error",
-    #       created_at: 2018-01-01T13:22Z
-    #     },
-    #   },
-    #   events: {
-    #     <event_id>: {
-    #       action: Init,
-    #       input: ...,
-    #       created_at: ...,
-    #       updated_at: ...,
-    #       attempts_count: 2,
-    #     },
-    #     <event_id>: {...},
-    #   }
-    # }
-    def clear
-      self.store = subject.send(
-        "#{field}=",
-        "status" => nil,
-        "state" => {},
-        "events" => {},
-        "errors" => {}
-      )
-    end
-
-    def clear!
-      clear
-      subject.update_attributes!(field => store)
+    def reset!
+      reset
+      save!
     end
 
     def status
@@ -71,21 +48,38 @@ module DirtyPipeline
 
     def commit!(event)
       store["status"] = event.destination if event.destination
-      require'pry';binding.pry unless event.changes.respond_to?(:to_h)
       store["state"].merge!(event.changes) unless event.changes.to_h.empty?
       store["errors"][event.id] = event.error unless event.error.to_h.empty?
       store["events"][event.id] = event.data unless event.data.to_h.empty?
-      subject.assign_attributes(field => store)
-      subject.save!
+      save!
     end
 
-    def processing_event
-      find_event(transactions_queue.processing_event.id)
+    def find_event(event_id)
+      return unless (found_event = store.dig("events", event_id))
+      Event.new(data: found_event, error: store.dig("errors", event_id))
     end
 
-    def find_event(event_id)
-      return unless (found_event = store["events"][event_id])
-      Event.new(data: found_event, error: store["errors"][event_id])
+    private
+
+    def valid_store?
+      (store.keys & %w(status events errors state)).size.eql?(4)
+    end
+
+    def save!
+      subject.send("#{field}=", store)
+      subject.save!
+    end
+
+    def reset
+      @store = subject.send(
+        "#{field}=",
+        {
+          "status" => nil,
+          "state" => {},
+          "events" => {},
+          "errors" => {}
+        }
+      )
     end
   end
 end

data/lib/dirty_pipeline/transaction.rb

@@ -1,56 +1,52 @@
 module DirtyPipeline
   class Transaction
-    attr_reader :locker, :storage, :subject, :pipeline, :queue, :event
-    def initialize(pipeline, queue, event)
+    attr_reader :locker, :storage, :subject, :pipeline, :event
+    def initialize(pipeline, event)
       @pipeline = pipeline
       @subject = pipeline.subject
       @storage = pipeline.storage
-      @queue = queue
       @event = event
     end
 
-    def retry
-      event.attempt_retry!
-      pipeline.schedule_cleanup
-
-      with_transaction { |*targs| yield(*targs) }
-    end
-
     def call
-      # return unless queue.event_in_progress?(event)
-
       event.start!
-      pipeline.schedule_cleanup
+      with_transaction { |*targs| yield(*targs) }
+    end
 
+    def retry
+      event.attempt_retry!
       with_transaction { |*targs| yield(*targs) }
     end
 
     private
 
     def with_transaction
+      pipeline.schedule_cleanup
+
       destination, action, max_attempts_count =
         pipeline.find_transition(event.transition)
                 .values_at(:to, :action, :attempts)
 
       storage.commit!(event)
 
-      # status.action_pool.unshift(action)
       subject.transaction(requires_new: true) do
-        raise ActiveRecord::Rollback if catch(:abort_transaction) do
-          yield(destination, action, *event.args); nil
-        end
+        with_abort_handling { yield(destination, action, *event.args) }
       end
     rescue => exception
       event.link_exception(exception)
       if max_attempts_count.to_i > event.attempts_count
         event.retry!
         pipeline.schedule_retry
-      else
-        pipeline.schedule_cleanup
       end
       raise
     ensure
       storage.commit!(event)
     end
+
+    def with_abort_handling
+      return unless catch(:abort_transaction) { yield; nil }
+      event.abort! unless event.abort?
+      raise ActiveRecord::Rollback
+    end
   end
 end

data/lib/dirty_pipeline/transition.rb

@@ -1,11 +1,11 @@
 module DirtyPipeline
   class Transition
     def Abort()
-      throw :abort, true
+      throw :fail_operation, :abort
     end
 
     def Error(error)
-      throw :fail_with_error, error
+      throw :fail_operation, error
     end
 
     def Success(changes = nil)
@@ -15,8 +15,8 @@ module DirtyPipeline
     def self.finalize(*args, **kwargs)
       event, pipeline, *args = args
       instance = new(event, *args, **kwargs)
-      pipeline.railway.switch_to(:call)
       return unless instance.respond_to?(:finalize)
+      pipeline.railway.switch_to(:call)
       instance.finalize(pipeline.subject)
     end
 
@@ -31,8 +31,10 @@ module DirtyPipeline
       event, pipeline, *args = args
       instance = new(event, *args, **kwargs)
       pipeline.railway[:undo] << event if instance.respond_to?(:undo)
-      pipeline.railway[:finalize] << event if instance.respond_to?(:finalize)
-      pipeline.railway.switch_to(:finalize) if instance.respond_to?(:finalize)
+      if instance.respond_to?(:finalize)
+        pipeline.railway[:finalize] << event
+        pipeline.railway.switch_to(:finalize)
+      end
       new(event, *args, **kwargs).call(pipeline.subject)
     end
 

data/lib/dirty_pipeline/version.rb

@@ -1,3 +1,3 @@
 module DirtyPipeline
-  VERSION = "0.5.0"
+  VERSION = "0.6.0"
 end

metadata

@@ -1,15 +1,57 @@
 --- !ruby/object:Gem::Specification
 name: dirty_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Sergey Dolganov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-02 00:00:00.000000000 Z
+date: 2018-09-03 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: sidekiq
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: nanoid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.2.0
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -52,6 +94,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.2'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.11'
 description: Simple state machine designed for non-pure transitions. E.g. for wizard-like
   systems with a lot of external API calls.
 email: