dirty_pipeline 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a3b3d599d44c9b6d8db7dee9fd91ebbebc77b8d25df0dd12c09f4e050a9f20af
-  data.tar.gz: d0ad5ae7de3e9e6d5fdc295a9d039cfcb52c555787cfac7b1348a25eb4a1ca88
+  metadata.gz: c67b4adb4c6e578a8add63619a5836d1d9d820c1b4c07fd53f2dd1877f1851d0
+  data.tar.gz: 94d3c03c15d6e74079f3e64b985392f88b1c07247c394de1bf58e2feb1b94693
 SHA512:
-  metadata.gz: d20cdfe6359c61edf2d6f5c0e0332f7697b25d6fedc4c3165b5051996ad63465379b6170bb4e97d495c1cb57e920517c8442989a53d3366593243028e0a3488e
-  data.tar.gz: b08171b0290fe36d2c2dff8c5d9658be808f22e27280eabb5f35c79ff368cb4762fbc4b7e38686d06a0461dc51cd8b6bd1251eaa2921b3e42c30ea205b524831
+  metadata.gz: 52c62a8e38f6ab759bf7368b0f32e86008894b48e4cc4a4fd93643d84848c436fb8b2aa93d200faa29e98431bc8136805b45fa76ba4d76e72cd262450a5f16da
+  data.tar.gz: 0e83e12bdfb27820726ebd789b0d6226df9cdce1256d3a9cd5e4137656ef83ec8ccbaaf8cb6ccc12d4727c588bc5b8676760b3712df8bb601505301d92ba77e1

data/lib/dirty_pipeline.rb

@@ -1,5 +1,13 @@
 require "dirty_pipeline/version"
 
 module DirtyPipeline
-  # Your code goes here...
+  require_relative "dirty_pipeline/locker.rb"
+  require_relative "dirty_pipeline/storage.rb"
+  require_relative "dirty_pipeline/transition.rb"
+  require_relative "dirty_pipeline/base.rb"
+
+  # This method should yield raw Redis connection
+  def self.with_redis
+    fail NotImplementedError
+  end
 end

data/lib/dirty_pipeline/base.rb

@@ -0,0 +1,272 @@
+module DirtyPipeline
+  class Base
+    DEFAULT_RETRY_DELAY = 5 * 60 # 5 minutes
+    DEFAULT_CLEANUP_DELAY = 60 * 60 * 24 # 1 day
+    RESERVED_STATUSES = [
+      Storage::FAILED_STATUS,
+      Storage::PROCESSING_STATUS,
+      Storage::RETRY_STATUS,
+      Locker::CLEAN,
+    ]
+
+    class ReservedStatusError < StandardError; end
+    class InvalidTransition < StandardError; end
+
+    class << self
+      def find_subject(*args)
+        fail NotImplemented
+      end
+
+      attr_reader :transitions_map
+      def inherited(child)
+        child.instance_variable_set(:@transitions_map, Hash.new)
+      end
+      # PG JSONB column
+      # {
+      #   status: :errored,
+      #   state: {
+      #     field: "value",
+      #   },
+      #   errors: [
+      #     {
+      #       error: "RuPost::API::Error",
+      #       error_message: "Timeout error",
+      #       created_at: 2018-01-01T13:22Z
+      #     },
+      #   ],
+      #   events: [
+      #     {
+      #       action: Init,
+      #       input: ...,
+      #       created_at: ...,
+      #       updated_at: ...,
+      #       attempts_count: 2,
+      #     },
+      #     {...},
+      #   ]
+      # }
+      attr_accessor :pipeline_storage, :retry_delay, :cleanup_delay
+
+      def transition(action, from:, to:, name: action.to_s, attempts: 1)
+        raise ReservedStatusError unless valid_statuses?(from, to)
+        @transitions_map[name] = {
+          action: action,
+          from: Array(from).map(&:to_s),
+          to: to.to_s,
+          attempts: attempts,
+        }
+      end
+
+      private
+
+      def valid_statuses?(from, to)
+        ((Array(to) + Array(from)) & RESERVED_STATUSES).empty?
+      end
+    end
+
+    def enqueue(transition_name, *args)
+      Shipping::PipelineWorker.perform_async(
+        "enqueued_pipeline" => self.class.to_s,
+        "find_subject_args" => find_subject_args,
+        "transition_args" => args.unshift(transition_name),
+      )
+    end
+
+    def reset!
+      storage.reset_pipeline_status!
+    end
+
+    attr_reader :subject, :error, :storage
+    def initialize(subject)
+      @subject = subject
+      @storage = Storage.new(subject, self.class.pipeline_storage)
+      @locker = Locker.new(@subject, @storage)
+    end
+
+    def call(*args)
+      return self if succeeded == false
+      self.succeeded = nil
+      after_commit = nil
+
+      # transaction with support of external calls
+      transaction(*args) do |destination, action, *transition_args|
+        output = {}
+        fail_cause = nil
+
+        output, *after_commit = catch(:success) do
+          fail_cause = catch(:fail_with_error) do
+            return Abort() if catch(:abort) do
+              throw :success, action.(subject, *transition_args)
+            end
+          end
+          nil
+        end
+
+        if fail_cause
+          ExpectedError(fail_cause)
+        else
+          Success(destination, output)
+        end
+      end
+
+      Array(after_commit).each { |cb| cb.call(subject) } if after_commit
+      self
+    end
+
+    def clear!
+      storage.clear!
+    end
+
+    def success?
+      succeeded
+    end
+
+    def when_success(callback = nil)
+      return self unless success?
+      if block_given?
+        yield(self)
+      else
+        callback.call(self)
+      end
+      self
+    end
+
+    def when_failed(callback = nil)
+      return self unless storage.failed?
+      if block_given?
+        yield(self)
+      else
+        callback.call(self)
+      end
+      self
+    end
+
+    def errored?
+      return if succeeded.nil?
+      ready? && !succeeded
+    end
+
+    def when_error(callback = nil)
+      return self unless errored?
+      if block_given?
+        yield(self)
+      else
+        callback.call(self)
+      end
+      self
+    end
+
+    def ready?
+      storage.pipeline_status.nil?
+    end
+
+    def when_processing(callback = nil)
+      return self unless storage.processing?
+      if block_given?
+        yield(self)
+      else
+        callback.call(self)
+      end
+      self
+    end
+
+    private
+
+    attr_writer :error
+    attr_reader :locker
+    attr_accessor :succeeded, :previous_status
+
+    def find_subject_args
+      subject.id
+    end
+
+    def retry_delay
+      self.class.retry_delay || DEFAULT_RETRY_DELAY
+    end
+
+    def cleanup_delay
+      self.class.cleanup_delay || DEFAULT_CLEANUP_DELAY
+    end
+
+    def Retry(error, *args)
+      storage.save_retry!(error)
+      Shipping::PipelineWorker.perform_in(
+        retry_delay,
+        "enqueued_pipeline" => self.class.to_s,
+        "find_subject_args" => find_subject_args,
+        "retry" => true,
+      )
+    end
+
+    def ExpectedError(cause)
+      self.error = cause
+      storage.fail_event!
+      self.succeeded = false
+    end
+
+    def Error(error)
+      storage.save_exception!(error)
+      self.error = error
+      self.succeeded = false
+    end
+
+    def Abort()
+      self.succeeded = false
+      throw :abort_transaction, true
+    end
+
+    def Success(destination, output)
+      storage.complete!(output, destination)
+      self.succeeded = true
+    end
+
+    def try_again?(max_attempts_count)
+      return unless max_attempts_count
+      storage.last_event["attempts_count"].to_i < max_attempts_count
+    end
+
+    def find_transition(name)
+      if (const_name = self.class.const_get(name) rescue nil)
+        name = const_name.to_s
+      end
+      self.class.transitions_map.fetch(name.to_s).tap do |from:, **kwargs|
+        next if from == Array(storage.status)
+        next if from.include?(storage.status.to_s)
+        raise InvalidTransition, "from `#{storage.status}` by `#{name}`"
+      end
+    end
+
+    def schedule_cleanup
+      Shipping::PipelineWorker.perform_in(
+        cleanup_delay,
+        "enqueued_pipeline" => self.class.to_s,
+        "find_subject_args" => find_subject_args,
+        "transition_args" => [Locker::CLEAN],
+      )
+    end
+
+    def transaction(*args)
+      locker.with_lock(*args) do |transition, *transition_args|
+        begin
+          schedule_cleanup
+          destination, action, max_attempts_count =
+            find_transition(transition).values_at(:to, :action, :attempts)
+
+          subject.transaction(requires_new: true) do
+            raise ActiveRecord::Rollback if catch(:abort_transaction) do
+              yield(destination, action, *transition_args); nil
+            end
+          end
+        rescue => error
+          if try_again?(max_attempts_count)
+            Retry(error)
+          else
+            # FIXME: Somehow :error is a Hash, all the time
+            Error(error)
+          end
+          raise
+        end
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/locker.rb

@@ -0,0 +1,111 @@
+module DirtyPipeline
+  class Locker
+    CLEAN = "clean".freeze
+
+    attr_reader :storage, :subject
+    def initialize(subject, storage)
+      @subject = subject
+      @storage = storage
+    end
+
+    class Normal
+      attr_reader :locker, :transition, :transition_args
+      def initialize(locker, args)
+        @locker = locker
+        @transition = args.shift
+        @transition_args = args
+      end
+
+      # NORMAL MODE
+      # if state is PROCESSING_STATE - finish
+      # if state is FAILED_STATE - finish
+      # otherwise - start
+      def skip_any_action?
+        [
+          Storage::PROCESSING_STATUS,
+          Storage::FAILED_STATUS,
+        ].include?(locker.storage.pipeline_status)
+      end
+
+      def start!
+        locker.storage.start!(transition, transition_args)
+      end
+
+      def lock!
+        return if skip_any_action?
+        start!
+        begin
+          yield(transition, *transition_args)
+        ensure
+          if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
+            locker.storage.reset_pipeline_status!
+          end
+        end
+      rescue
+        if locker.storage.pipeline_status == Storage::PROCESSING_STATUS
+          locker.storage.commit_pipeline_status!(Storage::FAILED_STATUS)
+        end
+        raise
+      end
+    end
+
+    class Retry < Normal
+      def initialize(locker, _args)
+        @locker = locker
+        @transition = locker.storage.last_event["transition"]
+        @transition_args = locker.storage.last_event["input"]
+      end
+
+      # RETRY MODE
+      # if state is not RETRY_STATE - finish
+      # if state is RETRY_STATE - start
+      def skip_any_action?
+        storage.status != Storage::RETRY_STATUS
+      end
+
+      def start!
+        locker.storage.start_retry!
+      end
+    end
+
+    # run in time much more then time to process an item
+    class Clean < Retry
+      ONE_DAY = 60 * 60 * 24
+
+      def skip_any_action?
+        return true if storage.status != Storage::PROCESSING_STATUS
+        started_less_then_a_day_ago?
+      end
+
+      def started_less_then_a_day_ago?
+        return unless (updated_at = locker.storage.last_event["updated_at"])
+        updated_at > one_day_ago
+      end
+
+      def one_day_ago
+        Time.now - ONE_DAY
+      end
+    end
+
+    def with_lock(*args)
+      lock!(*args) do |transition, *transition_args|
+        yield(transition, *transition_args)
+      end
+    end
+
+    def lock!(*args)
+      locker_klass(*args).new(self, args).lock! { |*largs| yield(*largs) }
+    end
+
+    def locker_klass(transition, *)
+      case transition
+      when Storage::RETRY_STATUS
+        Retry
+      when CLEAN
+        Clean
+      else
+        Normal
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/storage.rb

@@ -0,0 +1,193 @@
+module DirtyPipeline
+  class Storage
+    FAILED_STATUS = "failed".freeze
+    RETRY_STATUS = "retry".freeze
+    PROCESSING_STATUS = "processing".freeze
+    class InvalidPipelineStorage < StandardError; end
+
+    attr_reader :subject, :field
+    attr_accessor :store
+    alias :to_h :store
+    def initialize(subject, field)
+      @subject = subject
+      @field = field
+      init_store(field)
+    end
+
+    def init_store(store_field)
+      self.store = subject.send(store_field).to_h
+      clear! if store.empty?
+      return if (store.keys & %w(status events errors state)).size == 4
+      raise InvalidPipelineStorage, store
+    end
+
+    def clear
+      self.store = subject.send(
+        "#{field}=",
+        "status" => nil,
+        "pipeline_status" => nil,
+        "state" => {},
+        "events" => [],
+        "errors" => [],
+      )
+      DirtyPipeline.with_redis { |r| r.del(pipeline_status_key) }
+    end
+
+    def clear!
+      clear
+      commit!
+    end
+
+    def start!(transition, args)
+      events << {
+        "transition" => transition,
+        "args" => args,
+        "created_at" => Time.now
+      }
+      increment_attempts_count
+      self.pipeline_status = PROCESSING_STATUS
+      # self.status = "processing", should be set by Locker
+      commit!
+    end
+
+    def start_retry!
+      last_event.merge!(updated_at: Time.now)
+      increment_attempts_count
+      self.pipeline_status = PROCESSING_STATUS
+      # self.status = "processing", should be set by Locker
+      commit!
+    end
+
+    def complete!(output, destination)
+      store["status"] = destination
+      state.merge!(output)
+      last_event.merge!(
+        "output" => output,
+        "updated_at" => Time.now,
+        "success" => true,
+      )
+      commit!
+    end
+
+    def fail_event!
+      fail_event
+      commit!
+    end
+
+    def fail_event
+      last_event["failed"] = true
+    end
+
+    def status
+      store["status"]
+    end
+
+    def pipeline_status_key
+      "pipeline-status:#{subject.class}:#{subject.id}:#{field}"
+    end
+
+    def pipeline_status=(value)
+      DirtyPipeline.with_redis do |r|
+        if value
+          r.set(pipeline_status_key, value)
+        else
+          r.del(pipeline_status_key)
+        end
+      end
+      store["pipeline_status"] = value
+    end
+
+    def commit_pipeline_status!(value = nil)
+      self.pipeline_status = value
+      commit!
+    end
+    alias :reset_pipeline_status! :commit_pipeline_status!
+
+    def pipeline_status
+      DirtyPipeline.with_redis do |r|
+        store["pipeline_status"] = r.get(pipeline_status_key)
+      end
+      store.fetch("pipeline_status")
+    end
+
+    def state
+      store.fetch("state")
+    end
+
+    def events
+      store.fetch("events")
+    end
+
+    def last_event
+      events.last.to_h
+    end
+
+    def last_event_error(event_idx = nil)
+      event = events[event_idx] if event_idx
+      event ||= last_event
+      errors[event["error_idx"]].to_h
+    end
+
+    def errors
+      store.fetch("errors")
+    end
+
+    def last_error
+      errors.last.to_h
+    end
+
+    def increment_attempts_count
+      last_event.merge!(
+        "attempts_count" => last_event["attempts_count"].to_i + 1
+      )
+    end
+
+    def increment_attempts_count!
+      increment_attempts_count
+      commit!
+    end
+
+    def save_retry(error)
+      save_error(error)
+      self.pipeline_status = RETRY_STATUS
+    end
+
+    def save_retry!(error)
+      save_retry(error)
+      commit!
+    end
+
+    def save_exception(exception)
+      errors << {
+        "error" => exception.class.to_s,
+        "error_message" => exception.message,
+        "created_at" => Time.current,
+      }
+      last_event["error_idx"] = errors.size - 1
+      fail_event
+      self.pipeline_status = FAILED_STATUS
+    end
+
+    def save_exception!(error)
+      save_exception(error)
+      commit!
+    end
+
+    def commit!
+      subject.assign_attributes(field => store)
+      subject.save!
+    end
+
+    def ready?
+      storage.pipeline_status.nil?
+    end
+
+    def failed?
+      pipeline_status == FAILED_STATUS
+    end
+
+    def processing?
+      [PROCESSING_STATUS, RETRY_STATUS].include?(pipeline_status)
+    end
+  end
+end

data/lib/dirty_pipeline/transition.rb

@@ -0,0 +1,25 @@
+module DirtyPipeline
+  class Transition
+    def Abort()
+      throw :abort, true
+    end
+
+    def Error(error)
+      throw :fail_with_error, error
+    end
+
+    def Success(output = nil, after_commit: nil, &block)
+      result = [output.to_h]
+      after_commit = Array(after_commit) << block if block_given?
+      result += Array(after_commit) if after_commit
+      throw :success, result
+    end
+
+    def self.call(*args, **kwargs)
+      subject = args.shift
+      instance = new(*args, **kwargs)
+      instance.compensate(subject) if instance.respond_to?(:compensate)
+      instance.call(subject)
+    end
+  end
+end

data/lib/dirty_pipeline/version.rb

@@ -1,3 +1,3 @@
 module DirtyPipeline
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dirty_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Sergey Dolganov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-21 00:00:00.000000000 Z
+date: 2018-08-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -71,6 +71,10 @@ files:
 - bin/setup
 - dirty_pipeline.gemspec
 - lib/dirty_pipeline.rb
+- lib/dirty_pipeline/base.rb
+- lib/dirty_pipeline/locker.rb
+- lib/dirty_pipeline/storage.rb
+- lib/dirty_pipeline/transition.rb
 - lib/dirty_pipeline/version.rb
 homepage: https://github.com/sclinede/dirty_pipeline
 licenses: