dirty_pipeline 0.7.1 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f1d848ae9c9559b571abc4a8561be9f7deb9d7aaa4977f5c5c7993fd698881cd
-  data.tar.gz: 95d0fe52b224ea34310a102cf2dd1d31cafb298cb8c9a25065b9ee41f819a238
+  metadata.gz: 9f3fd5ee73f6187ea39daa3d7b88371c84f3c6ca848b7ba65f58b7c8876c55b5
+  data.tar.gz: ab18cf256c02e3851a87c093523bffe204bc25bba6a31be8027a923ddbebaa69
 SHA512:
-  metadata.gz: ae206aaf6b7b7cda711467775e912dfa888f3024d4fde8e4516eab8c07a9f6d8d5e983867a415ddb9765506acb1e3a4d7cb64ec9af828bb54b2d31404c587280
-  data.tar.gz: 64f3e7ca995413684d0b77676c178877b893419775433faf2e855c29538feff1833f39f26671d4b9577ef29e660a9c50417f597cdada71109e51bcce36cf9631
+  metadata.gz: a818876d704b6dbb40ec6a0bad6ac45fb38a40ad87fce41859ea5cc2a29ddb8d72067aefdb63d16d888c6cef0fcd840f3b65ca00b0fc810133b081ed10bd25f7
+  data.tar.gz: e6e014a6f8b37f55a688c5797c49287642531d4cc3dc2373688936b1e9bc0c2e7263947e9cbdcf6e9e152294cc0cf6c34f24e5835bad782c5bd497c7960f897e

data/dirty_pipeline.gemspec

@@ -24,6 +24,7 @@ Gem::Specification.new do |spec|
   # temporary dependency
   spec.add_runtime_dependency "sidekiq", "~> 5.0"
   spec.add_runtime_dependency "redis", "~> 4.0"
+  spec.add_runtime_dependency "pg", "~> 1.0"
 
   spec.add_development_dependency "bundler", "~> 1.16"
   spec.add_development_dependency "rake", "~> 10.0"

data/lib/dirty_pipeline.rb

@@ -4,12 +4,21 @@ require "securerandom"
 module DirtyPipeline
   require_relative "dirty_pipeline/ext/camelcase.rb"
   require_relative "dirty_pipeline/status.rb"
-  require_relative "dirty_pipeline/storage.rb"
   require_relative "dirty_pipeline/worker.rb"
   require_relative "dirty_pipeline/transaction.rb"
   require_relative "dirty_pipeline/event.rb"
-  require_relative "dirty_pipeline/queue.rb"
-  require_relative "dirty_pipeline/railway.rb"
+
+  # Redis
+  require_relative "dirty_pipeline/redis/railway.rb"
+  require_relative "dirty_pipeline/redis/storage.rb"
+  require_relative "dirty_pipeline/redis/queue.rb"
+
+  # Postgres
+  require_relative "dirty_pipeline/pg.rb"
+  require_relative "dirty_pipeline/pg/railway.rb"
+  require_relative "dirty_pipeline/pg/storage.rb"
+  require_relative "dirty_pipeline/pg/queue.rb"
+
   require_relative "dirty_pipeline/base.rb"
   require_relative "dirty_pipeline/transition.rb"
 
@@ -17,4 +26,31 @@ module DirtyPipeline
   def self.with_redis
     fail NotImplementedError
   end
+
+  # This method should yield raw PG connection
+  def self.with_postgres
+    fail NotImplementedError
+  end
+
+  # def self.with_postgres
+  #   yield(ActiveRecord::Base.connection.raw_connection)
+  # ensure
+  #   ActiveRecord::Base.clear_active_connections!
+  # end
+
+  Queue = Redis::Queue
+  Storage = Redis::Storage
+  Railway = Redis::Railway
+
+  def self.create!(conn)
+    Queue.create!(conn)   if Queue.respond_to?(:create!)
+    Storage.create!(conn) if Storage.respond_to?(:create!)
+    Railway.create!(conn) if Railway.respond_to?(:create!)
+  end
+
+  def self.destroy!(conn)
+    Queue.destroy!(conn)   if Queue.respond_to?(:destroy!)
+    Storage.destroy!(conn) if Storage.respond_to?(:destroy!)
+    Railway.destroy!(conn) if Railway.respond_to?(:destroy!)
+  end
 end

data/lib/dirty_pipeline/event.rb

@@ -1,4 +1,5 @@
 require 'json'
+require 'time'
 
 module DirtyPipeline
   class Event
@@ -41,7 +42,7 @@ module DirtyPipeline
         "transaction_uuid" => @tx_id,
         "transition" => transition,
         "args" => args,
-        "created_at" => Time.now,
+        "created_at" => Time.now.utc.iso8601,
         "cache" => {},
         "attempts_count" => 1,
         "status" => NEW,
@@ -71,7 +72,7 @@ module DirtyPipeline
       @error = {
         "exception" => exception.class.to_s,
         "exception_message" => exception.message,
-        "created_at" => Time.now,
+        "created_at" => Time.now.utc.iso8601,
       }
       failure!
     end
@@ -81,7 +82,7 @@ module DirtyPipeline
     end
 
     def attempt_retry!
-      @data["updated_at"] = Time.now
+      @data["updated_at"] = Time.now.utc.iso8601
       @data["attempts_count"] = attempts_count + 1
     end
 
@@ -89,7 +90,7 @@ module DirtyPipeline
       @data.merge!(
         "destination" => destination,
         "changes" => changes,
-        "updated_at" => Time.now,
+        "updated_at" => Time.now.utc.iso8601,
         "status" => SUCCESS,
       )
     end

data/lib/dirty_pipeline/pg.rb

@@ -0,0 +1,12 @@
+module DirtyPipeline
+  module PG
+    module_function
+    def multi(pg_result)
+      pg_result.first&.values
+    end
+
+    def single(pg_result)
+      pg_result.first&.values&.first
+    end
+  end
+end

data/lib/dirty_pipeline/pg/queue.rb

@@ -0,0 +1,171 @@
+module DirtyPipeline
+  module PG
+    class Queue
+      # decoder = PG::TextDecoder::Array.new
+      # see https://stackoverflow.com/questions/34886260/how-do-you-decode-a-json-field-using-the-pg-gem
+      def self.create!(connection)
+        connection.exec <<~SQL
+          CREATE TABLE dp_active_events (
+            key TEXT CONSTRAINT primary_event_queues_key PRIMARY KEY,
+            payload TEXT,
+            created_at TIMESTAMP NOT NULL DEFAULT now()
+          );
+
+          CREATE SEQUENCE dp_event_queues_id_seq START 1;
+          CREATE TABLE dp_event_queues (
+            id BIGINT PRIMARY KEY DEFAULT nextval('dp_event_queues_id_seq'),
+            key TEXT NOT NULL,
+            payload TEXT,
+            created_at TIMESTAMP NOT NULL DEFAULT now()
+          );
+        SQL
+      end
+
+      def self.destroy!(connection)
+        connection.exec <<~SQL
+          DROP TABLE IF EXISTS dp_active_events;
+          DROP TABLE IF EXISTS dp_event_queues;
+          DROP SEQUENCE IF EXISTS dp_event_queues_id_seq;
+        SQL
+      end
+
+      def initialize(operation, subject_class, subject_id, transaction_id)
+        @root = "dirty-pipeline-queue:#{subject_class}:#{subject_id}:" \
+                "op_#{operation}:txid_#{transaction_id}"
+      end
+
+      def with_postgres(&block)
+        DirtyPipeline.with_postgres(&block)
+      end
+
+      DELETE_ACTIVE = <<~SQL
+        DELETE FROM dp_active_events WHERE key = $1;
+      SQL
+      DELETE_EVENTS = <<~SQL
+        DELETE FROM dp_event_queues WHERE key = $1;
+      SQL
+
+      def clear!
+        with_postgres do |c|
+          c.transaction do
+            c.exec(DELETE_ACTIVE, [active_event_key])
+            c.exec(DELETE_EVENTS, [events_queue_key])
+          end
+        end
+      end
+
+      SELECT_ALL_EVENTS = <<~SQL
+        SELECT payload FROM dp_event_queues WHERE key = $1 ORDER BY id DESC;
+      SQL
+      def to_a
+        with_postgres do |c|
+          c.exec(SELECT_ALL_EVENTS, [events_queue_key]).to_a.map! do |row|
+            unpack(row.values.first)
+          end
+        end
+      end
+
+      PUSH_EVENT = <<~SQL
+        INSERT INTO dp_event_queues (id, key, payload)
+        VALUES (-nextval('dp_event_queues_id_seq'), $1, $2);
+      SQL
+      def push(event)
+        with_postgres do |c|
+          c.transaction do
+            c.exec(PUSH_EVENT, [events_queue_key, pack(event)])
+          end
+        end
+
+        self
+      end
+      alias :<< :push
+
+      UNSHIFT_EVENT = <<~SQL
+        INSERT INTO dp_event_queues (key, payload) VALUES ($1, $2);
+      SQL
+      def unshift(event)
+        with_postgres do |c|
+          c.transaction do
+            c.exec(UNSHIFT_EVENT, [events_queue_key, pack(event)])
+          end
+        end
+        self
+      end
+
+      SELECT_LAST_EVENT = <<~SQL
+        SELECT id, payload FROM dp_event_queues
+        WHERE key = $1
+        ORDER BY id DESC LIMIT 1;
+      SQL
+      DELETE_EVENT = <<~SQL
+        DELETE FROM dp_event_queues WHERE key = $1 AND id = $2;
+      SQL
+      DELETE_ACTIVE_EVENT = <<~SQL
+        DELETE FROM dp_active_events WHERE key = $1;
+      SQL
+      SET_EVENT_ACTIVE = <<~SQL
+        INSERT INTO dp_active_events (key, payload) VALUES ($1, $2)
+        ON CONFLICT (key) DO UPDATE SET payload = EXCLUDED.payload;
+      SQL
+      def pop
+        with_postgres do |c|
+          c.transaction do
+            event_id, raw_event =
+              PG.multi(c.exec(SELECT_LAST_EVENT, [events_queue_key]))
+            if raw_event.nil?
+              c.exec(DELETE_ACTIVE_EVENT, [active_event_key])
+            else
+              c.exec(DELETE_EVENT, [events_queue_key, event_id])
+              c.exec(SET_EVENT_ACTIVE, [active_event_key, raw_event])
+            end
+            unpack(raw_event)
+          end
+        end
+      end
+
+      SELECT_ACTIVE_EVENT = <<~SQL
+        SELECT payload FROM dp_active_events WHERE key = $1;
+      SQL
+      def processing_event
+        with_postgres do |c|
+          raw_event = PG.single(
+            c.exec(SELECT_ACTIVE_EVENT, [active_event_key])
+          )
+          unpack(raw_event)
+        end
+      end
+
+      private
+
+      def pack(event)
+        JSON.dump(
+          "evid" => event.id,
+          "txid" => event.tx_id,
+          "transit" => event.transition,
+          "args" => event.args,
+        )
+      end
+
+      def unpack(packed_event)
+        return unless packed_event
+        unpacked_event = JSON.load(packed_event)
+        Event.new(
+          data: {
+            "uuid" => unpacked_event["evid"],
+            "transaction_uuid" => unpacked_event["txid"],
+            "transition" => unpacked_event["transit"],
+            "args" => unpacked_event["args"],
+          }
+        )
+      end
+
+      def events_queue_key
+        "#{@root}:events"
+      end
+
+      def active_event_key
+        "#{@root}:active"
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/pg/railway.rb

@@ -0,0 +1,143 @@
+module DirtyPipeline
+  module PG
+    class Railway
+      DEFAULT_OPERATIONS = %w(call undo finalize)
+
+      def self.create!(connection)
+        connection.exec <<~SQL
+          CREATE TABLE dp_active_operations (
+            key TEXT CONSTRAINT primary_dp_active_operations_key PRIMARY KEY,
+            name TEXT,
+            created_at TIMESTAMP NOT NULL DEFAULT now()
+          );
+          CREATE TABLE dp_active_transactions (
+            key TEXT CONSTRAINT primary_dp_active_tx_key PRIMARY KEY,
+            name TEXT,
+            created_at TIMESTAMP NOT NULL DEFAULT now()
+          );
+        SQL
+      end
+
+      def self.destroy!(connection)
+        connection.exec <<~SQL
+          DROP TABLE IF EXISTS dp_active_operations;
+          DROP TABLE IF EXISTS dp_active_transactions;
+        SQL
+      end
+
+      def initialize(subject, transaction_id)
+        @tx_id = transaction_id
+        @subject_class = subject.class.to_s
+        @subject_id = subject.id.to_s
+        @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:"
+        @queues = Hash[
+          DEFAULT_OPERATIONS.map do |operation|
+            [operation, create_queue(operation)]
+          end
+        ]
+      end
+
+      DELETE_OPERATION = <<~SQL
+        DELETE FROM dp_active_operations WHERE key = $1;
+      SQL
+      DELETE_TRANSACTION = <<~SQL
+        DELETE FROM dp_active_transactions WHERE key = $1;
+      SQL
+      def clear!
+        @queues.values.each(&:clear!)
+        DirtyPipeline.with_postgres do |c|
+          c.transaction do
+            c.exec DELETE_OPERATION, [active_operation_key]
+            c.exec DELETE_TRANSACTION, [active_transaction_key]
+          end
+        end
+      end
+
+      def next
+        return if other_transaction_in_progress?
+        start_transaction! unless running_transaction
+
+        queue.pop.tap { |event| finish_transaction! if event.nil? }
+      end
+
+      def queue(operation_name = active)
+        @queues.fetch(operation_name.to_s) do
+          @queues.store(operation_name, create_queue(operation_name))
+        end
+      end
+      alias :[] :queue
+
+      SWITCH_OPERATION = <<~SQL
+        INSERT INTO dp_active_operations (key, name) VALUES ($1, $2)
+        ON CONFLICT (key)
+        DO UPDATE SET name = EXCLUDED.name;
+      SQL
+      def switch_to(name)
+        raise ArgumentError unless DEFAULT_OPERATIONS.include?(name.to_s)
+        return if name.to_s == active
+
+        DirtyPipeline.with_postgres do |c|
+          c.transaction do
+            c.exec(SWITCH_OPERATION, [active_operation_key, name])
+          end
+        end
+      end
+
+      SELECT_OPERATION = <<~SQL
+        SELECT name FROM dp_active_operations WHERE key = $1;
+      SQL
+      def active
+        DirtyPipeline.with_postgres do |c|
+          PG.single c.exec(SELECT_OPERATION, [active_operation_key])
+        end
+      end
+      alias :operation :active
+
+      SELECT_TRANSACTION = <<~SQL
+        SELECT name FROM dp_active_transactions WHERE key = $1;
+      SQL
+      def running_transaction
+        DirtyPipeline.with_postgres do |c|
+          PG.single c.exec(SELECT_TRANSACTION, [active_transaction_key])
+        end
+      end
+
+      def other_transaction_in_progress?
+        return false if running_transaction.nil?
+        running_transaction != @tx_id
+      end
+
+      private
+
+      def create_queue(operation_name)
+        Queue.new(operation_name, @subject_class, @subject_id, @tx_id)
+      end
+
+      def active_transaction_key
+        "#{@root}:active_transaction"
+      end
+
+      def active_operation_key
+        "#{@root}:active_operation"
+      end
+
+      SWITCH_TRANSACTION = <<~SQL
+        INSERT INTO dp_active_transactions (key, name) VALUES ($1, $2)
+        ON CONFLICT (key)
+        DO UPDATE SET name = EXCLUDED.name;
+      SQL
+      def start_transaction!
+        switch_to(DEFAULT_OPERATIONS.first) unless active
+        DirtyPipeline.with_postgres do |c|
+          c.transaction do
+            c.exec(SWITCH_TRANSACTION, [active_transaction_key, @tx_id])
+          end
+        end
+      end
+
+      def finish_transaction!
+        clear! if running_transaction == @tx_id
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/pg/storage.rb

@@ -0,0 +1,112 @@
+module DirtyPipeline
+  # Storage structure
+  # {
+  #   status: :errored,
+  #   state: {
+  #     field: "value",
+  #   }
+  # }
+  module PG
+    class Storage
+      class InvalidPipelineStorage < StandardError; end
+
+      def self.create!(connection)
+        connection.exec <<~SQL
+          CREATE TABLE dp_events_store (
+            uuid TEXT CONSTRAINT primary_active_operations_key PRIMARY KEY,
+            context TEXT NOT NULL,
+            data TEXT,
+            error TEXT,
+            created_at TIMESTAMP NOT NULL DEFAULT now()
+          );
+        SQL
+      end
+
+      def self.destroy!(connection)
+        connection.exec <<~SQL
+          DROP TABLE IF EXISTS dp_events_store;
+        SQL
+      end
+
+      attr_reader :subject, :field, :store, :subject_key
+      alias :to_h :store
+      def initialize(subject, field)
+        @subject = subject
+        @field = field
+        @store = subject.send(@field).to_h
+        reset if @store.empty?
+        @subject_key = "#{subject.class}:#{subject.id}"
+        raise InvalidPipelineStorage, store unless valid_store?
+      end
+
+      def reset!
+        reset
+        save!
+      end
+
+      def status
+        store["status"]
+      end
+
+      SAVE_EVENT = <<~SQL
+        INSERT INTO dp_events_store (uuid, context, data, error)
+        VALUES ($1, $2, $3, $4)
+        ON CONFLICT (uuid)
+        DO UPDATE SET data = EXCLUDED.data, error = EXCLUDED.error;
+      SQL
+      def commit!(event)
+        store["status"] = event.destination  if event.destination
+        store["state"].merge!(event.changes) unless event.changes.to_h.empty?
+        DirtyPipeline.with_postgres do |c|
+          data, error = {}, {}
+          data = event.data.to_h if event.data.respond_to?(:to_h)
+          error = event.error.to_h if event.error.respond_to?(:to_h)
+          c.transaction do
+            c.exec(
+              SAVE_EVENT,
+              [event.id, subject_key, JSON.dump(data), JSON.dump(error)]
+            )
+          end
+        end
+        save!
+      end
+
+      FIND_EVENT = <<-SQL
+        SELECT data, error FROM dp_events_store
+        WHERE uuid = $1 AND context = $2;
+      SQL
+      def find_event(event_id)
+        DirtyPipeline.with_postgres do |c|
+          found_event, found_error =
+            PG.multi(c.exec(FIND_EVENT, [event_id, subject_key]))
+          return unless found_event
+          Event.new(
+            data: JSON.parse(found_event), error: JSON.parse(found_error)
+          )
+        end
+      end
+
+      private
+
+      def valid_store?
+        (store.keys & %w(status state)).size.eql?(2)
+      end
+
+      # FIXME: save! - configurable method
+      def save!
+        subject.send("#{field}=", store)
+        subject.save!
+      end
+
+      def reset
+        @store = subject.send(
+          "#{field}=",
+          {
+            "status" => nil,
+            "state" => {},
+          }
+        )
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/redis/queue.rb

@@ -0,0 +1,80 @@
+module DirtyPipeline
+  module Redis
+    class Queue
+      def initialize(operation, subject_class, subject_id, transaction_id)
+        @root = "dirty-pipeline-queue:#{subject_class}:#{subject_id}:" \
+                "op_#{operation}:txid_#{transaction_id}"
+      end
+
+      def clear!
+        DirtyPipeline.with_redis do |r|
+          r.del active_event_key
+          r.del events_queue_key
+        end
+      end
+
+      def to_a
+        DirtyPipeline.with_redis do |r|
+          r.lrange(events_queue_key, 0, -1).map! do |packed_event|
+            unpack(packed_event)
+          end
+        end
+      end
+
+      def push(event)
+        DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
+        self
+      end
+      alias :<< :push
+
+      def unshift(event)
+        DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
+        self
+      end
+
+      def pop
+        DirtyPipeline.with_redis do |r|
+          data = r.lpop(events_queue_key)
+          data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
+          unpack(data)
+        end
+      end
+
+      def processing_event
+        DirtyPipeline.with_redis { |r| unpack(r.get(active_event_key)) }
+      end
+
+      private
+
+      def pack(event)
+        JSON.dump(
+          "evid" => event.id,
+          "txid" => event.tx_id,
+          "transit" => event.transition,
+          "args" => event.args,
+        )
+      end
+
+      def unpack(packed_event)
+        return unless packed_event
+        unpacked_event = JSON.load(packed_event)
+        Event.new(
+          data: {
+            "uuid" => unpacked_event["evid"],
+            "transaction_uuid" => unpacked_event["txid"],
+            "transition" => unpacked_event["transit"],
+            "args" => unpacked_event["args"],
+          }
+        )
+      end
+
+      def events_queue_key
+        "#{@root}:events"
+      end
+
+      def active_event_key
+        "#{@root}:active"
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/redis/railway.rb

@@ -0,0 +1,86 @@
+module DirtyPipeline
+  module Redis
+    class Railway
+      DEFAULT_OPERATIONS = %w(call undo finalize)
+
+      def initialize(subject, transaction_id)
+        @tx_id = transaction_id
+        @subject_class = subject.class.to_s
+        @subject_id = subject.id.to_s
+        @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:"
+        @queues = Hash[
+          DEFAULT_OPERATIONS.map do |operation|
+            [operation, create_queue(operation)]
+          end
+        ]
+      end
+
+      def clear!
+        @queues.values.each(&:clear!)
+        DirtyPipeline.with_redis do |r|
+          r.multi do |mr|
+            mr.del(active_operation_key)
+            mr.del(active_transaction_key)
+          end
+        end
+      end
+
+      def next
+        return if other_transaction_in_progress?
+        start_transaction! unless running_transaction
+
+        queue.pop.tap { |event| finish_transaction! if event.nil? }
+      end
+
+      def queue(operation_name = active)
+        @queues.fetch(operation_name.to_s) do
+          @queues.store(operation_name, create_queue(operation_name))
+        end
+      end
+      alias :[] :queue
+
+      def switch_to(name)
+        raise ArgumentError unless DEFAULT_OPERATIONS.include?(name.to_s)
+        return if name.to_s == active
+        DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
+      end
+
+      def active
+        DirtyPipeline.with_redis { |r| r.get(active_operation_key) }
+      end
+      alias :operation :active
+
+      def running_transaction
+        DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
+      end
+
+      def other_transaction_in_progress?
+        return false if running_transaction.nil?
+        running_transaction != @tx_id
+      end
+
+      private
+
+      def create_queue(operation_name)
+        Queue.new(operation_name, @subject_class, @subject_id, @tx_id)
+      end
+
+      def active_transaction_key
+        "#{@root}:active_transaction"
+      end
+
+      def active_operation_key
+        "#{@root}:active_operation"
+      end
+
+      def start_transaction!
+        switch_to(DEFAULT_OPERATIONS.first) unless active
+        DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
+      end
+
+      def finish_transaction!
+        clear! if running_transaction == @tx_id
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/redis/storage.rb

@@ -0,0 +1,94 @@
+module DirtyPipeline
+  # Storage structure
+  # {
+  #   status: :errored,
+  #   state: {
+  #     field: "value",
+  #   },
+  #   errors: {
+  #     "<event_id>": {
+  #       error: "RuPost::API::Error",
+  #       error_message: "Timeout error",
+  #       created_at: 2018-01-01T13:22Z
+  #     },
+  #   },
+  #   events: {
+  #     <event_id>: {
+  #       transition: "Create",
+  #       args: ...,
+  #       changes: ...,
+  #       created_at: ...,
+  #       updated_at: ...,
+  #       attempts_count: 2,
+  #     },
+  #     <event_id>: {...},
+  #   }
+  # }
+  module Redis
+    class Storage
+      class InvalidPipelineStorage < StandardError; end
+
+      attr_reader :subject, :field, :store
+      alias :to_h :store
+      def initialize(subject, field)
+        @subject = subject
+        @field = field
+        @store = subject.send(@field).to_h
+        reset if @store.empty?
+        raise InvalidPipelineStorage, store unless valid_store?
+      end
+
+      def reset!
+        reset
+        save!
+      end
+
+      def status
+        store["status"]
+      end
+
+      def commit!(event)
+        store["status"] = event.destination     if event.destination
+        store["state"].merge!(event.changes)    unless event.changes.to_h.empty?
+
+        error = {}
+        error = event.error.to_h unless event.error.to_h.empty?
+        store["errors"][event.id] = error
+
+        data = {}
+        data = event.data.to_h unless event.data.to_h.empty?
+        store["events"][event.id] = data
+        save!
+      end
+
+      def find_event(event_id)
+        return unless (found_event = store.dig("events", event_id))
+        Event.new(data: found_event, error: store.dig("errors", event_id))
+      end
+
+      private
+
+      def valid_store?
+        (store.keys & %w(status events errors state)).size.eql?(4)
+      end
+
+      # FIXME: save! - configurable method
+      def save!
+        subject.send("#{field}=", store)
+        subject.save!
+      end
+
+      def reset
+        @store = subject.send(
+          "#{field}=",
+          {
+            "status" => nil,
+            "state" => {},
+            "events" => {},
+            "errors" => {}
+          }
+        )
+      end
+    end
+  end
+end

data/lib/dirty_pipeline/transaction.rb

@@ -19,7 +19,8 @@ module DirtyPipeline
       storage.commit!(event)
 
       # FIXME: make configurable, now - hardcoded to AR API
-      subject.transaction(requires_new: true) do
+      # subject.transaction(requires_new: true) do
+      subject.transaction do
         with_abort_handling { yield(destination, action, *event.args) }
       end
     rescue => exception

data/lib/dirty_pipeline/version.rb

@@ -1,3 +1,3 @@
 module DirtyPipeline
-  VERSION = "0.7.1"
+  VERSION = "0.8.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dirty_pipeline
 version: !ruby/object:Gem::Version
-  version: 0.7.1
+  version: 0.8.1
 platform: ruby
 authors:
 - Sergey Dolganov
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-10-08 00:00:00.000000000 Z
+date: 2018-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: sidekiq
@@ -38,6 +38,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -144,10 +158,14 @@ files:
 - lib/dirty_pipeline/base.rb
 - lib/dirty_pipeline/event.rb
 - lib/dirty_pipeline/ext/camelcase.rb
-- lib/dirty_pipeline/queue.rb
-- lib/dirty_pipeline/railway.rb
+- lib/dirty_pipeline/pg.rb
+- lib/dirty_pipeline/pg/queue.rb
+- lib/dirty_pipeline/pg/railway.rb
+- lib/dirty_pipeline/pg/storage.rb
+- lib/dirty_pipeline/redis/queue.rb
+- lib/dirty_pipeline/redis/railway.rb
+- lib/dirty_pipeline/redis/storage.rb
 - lib/dirty_pipeline/status.rb
-- lib/dirty_pipeline/storage.rb
 - lib/dirty_pipeline/transaction.rb
 - lib/dirty_pipeline/transition.rb
 - lib/dirty_pipeline/version.rb

data/lib/dirty_pipeline/queue.rb

@@ -1,78 +0,0 @@
-module DirtyPipeline
-  class Queue
-    def initialize(operation, subject_class, subject_id, transaction_id)
-      @root = "dirty-pipeline-queue:#{subject_class}:#{subject_id}:" \
-              "op_#{operation}:txid_#{transaction_id}"
-    end
-
-    def clear!
-      DirtyPipeline.with_redis do |r|
-        r.del active_event_key
-        r.del events_queue_key
-      end
-    end
-
-    def to_a
-      DirtyPipeline.with_redis do |r|
-        r.lrange(events_queue_key, 0, -1).map! do |packed_event|
-          unpack(packed_event)
-        end
-      end
-    end
-
-    def push(event)
-      DirtyPipeline.with_redis { |r| r.rpush(events_queue_key, pack(event)) }
-      self
-    end
-    alias :<< :push
-
-    def unshift(event)
-      DirtyPipeline.with_redis { |r| r.lpush(events_queue_key, pack(event)) }
-      self
-    end
-
-    def pop
-      DirtyPipeline.with_redis do |r|
-        data = r.lpop(events_queue_key)
-        data.nil? ? r.del(active_event_key) : r.set(active_event_key, data)
-        unpack(data)
-      end
-    end
-
-    def processing_event
-      DirtyPipeline.with_redis { |r| unpack(r.get(active_event_key)) }
-    end
-
-    private
-
-    def pack(event)
-      JSON.dump(
-        "evid" => event.id,
-        "txid" => event.tx_id,
-        "transit" => event.transition,
-        "args" => event.args,
-      )
-    end
-
-    def unpack(packed_event)
-      return unless packed_event
-      unpacked_event = JSON.load(packed_event)
-      Event.new(
-        data: {
-          "uuid" => unpacked_event["evid"],
-          "transaction_uuid" => unpacked_event["txid"],
-          "transition" => unpacked_event["transit"],
-          "args" => unpacked_event["args"],
-        }
-      )
-    end
-
-    def events_queue_key
-      "#{@root}:events"
-    end
-
-    def active_event_key
-      "#{@root}:active"
-    end
-  end
-end

data/lib/dirty_pipeline/railway.rb

@@ -1,84 +0,0 @@
-module DirtyPipeline
-  class Railway
-    DEFAULT_OPERATIONS = %w(call undo finalize)
-
-    def initialize(subject, transaction_id)
-      @tx_id = transaction_id
-      @subject_class = subject.class.to_s
-      @subject_id = subject.id.to_s
-      @root = "dirty-pipeline-rail:#{subject.class}:#{subject.id}:"
-      @queues = Hash[
-        DEFAULT_OPERATIONS.map do |operation|
-          [operation, create_queue(operation)]
-        end
-      ]
-    end
-
-    def clear!
-      @queues.values.each(&:clear!)
-      DirtyPipeline.with_redis do |r|
-        r.multi do |mr|
-          mr.del(active_operation_key)
-          mr.del(active_transaction_key)
-        end
-      end
-    end
-
-    def next
-      return if other_transaction_in_progress?
-      start_transaction! unless running_transaction
-
-      queue.pop.tap { |event| finish_transaction! if event.nil? }
-    end
-
-    def queue(operation_name = active)
-      @queues.fetch(operation_name.to_s) do
-        @queues.store(operation_name, create_queue(operation_name))
-      end
-    end
-    alias :[] :queue
-
-    def switch_to(name)
-      raise ArgumentError unless DEFAULT_OPERATIONS.include?(name.to_s)
-      return if name.to_s == active
-      DirtyPipeline.with_redis { |r| r.set(active_operation_key, name) }
-    end
-
-    def active
-      DirtyPipeline.with_redis { |r| r.get(active_operation_key) }
-    end
-    alias :operation :active
-
-    def running_transaction
-      DirtyPipeline.with_redis { |r| r.get(active_transaction_key) }
-    end
-
-    def other_transaction_in_progress?
-      return false if running_transaction.nil?
-      running_transaction != @tx_id
-    end
-
-    private
-
-    def create_queue(operation_name)
-      Queue.new(operation_name, @subject_class, @subject_id, @tx_id)
-    end
-
-    def active_transaction_key
-      "#{@root}:active_transaction"
-    end
-
-    def active_operation_key
-      "#{@root}:active_operation"
-    end
-
-    def start_transaction!
-      switch_to(DEFAULT_OPERATIONS.first) unless active
-      DirtyPipeline.with_redis { |r| r.set(active_transaction_key, @tx_id) }
-    end
-
-    def finish_transaction!
-      clear! if running_transaction == @tx_id
-    end
-  end
-end

data/lib/dirty_pipeline/storage.rb

@@ -1,86 +0,0 @@
-module DirtyPipeline
-  # Storage structure
-  # {
-  #   status: :errored,
-  #   state: {
-  #     field: "value",
-  #   },
-  #   errors: {
-  #     "<event_id>": {
-  #       error: "RuPost::API::Error",
-  #       error_message: "Timeout error",
-  #       created_at: 2018-01-01T13:22Z
-  #     },
-  #   },
-  #   events: {
-  #     <event_id>: {
-  #       transition: "Create",
-  #       args: ...,
-  #       changes: ...,
-  #       created_at: ...,
-  #       updated_at: ...,
-  #       attempts_count: 2,
-  #     },
-  #     <event_id>: {...},
-  #   }
-  # }
-  class Storage
-    class InvalidPipelineStorage < StandardError; end
-
-    attr_reader :subject, :field, :store
-    alias :to_h :store
-    def initialize(subject, field)
-      @subject = subject
-      @field = field
-      @store = subject.send(@field).to_h
-      reset if @store.empty?
-      raise InvalidPipelineStorage, store unless valid_store?
-    end
-
-    def reset!
-      reset
-      save!
-    end
-
-    def status
-      store["status"]
-    end
-
-    def commit!(event)
-      store["status"] = event.destination     if event.destination
-      store["state"].merge!(event.changes)    unless event.changes.to_h.empty?
-      store["errors"][event.id] = event.error unless event.error.to_h.empty?
-      store["events"][event.id] = event.data  unless event.data.to_h.empty?
-      save!
-    end
-
-    def find_event(event_id)
-      return unless (found_event = store.dig("events", event_id))
-      Event.new(data: found_event, error: store.dig("errors", event_id))
-    end
-
-    private
-
-    def valid_store?
-      (store.keys & %w(status events errors state)).size.eql?(4)
-    end
-
-    # FIXME: save! - configurable method
-    def save!
-      subject.send("#{field}=", store)
-      subject.save!
-    end
-
-    def reset
-      @store = subject.send(
-        "#{field}=",
-        {
-          "status" => nil,
-          "state" => {},
-          "events" => {},
-          "errors" => {}
-        }
-      )
-    end
-  end
-end