pub_sub_model_sync 0.5.8.2 → 1.0.beta

data/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: sh -c 'bundle exec rspec'
+    volumes:
+      - .:/app
+      - bundler_gems:/usr/local/bundle/
+
+volumes:
+  bundler_gems:

data/lib/pub_sub_model_sync.rb

@@ -10,8 +10,10 @@ require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'
 require 'pub_sub_model_sync/runner'
+require 'pub_sub_model_sync/transaction'
 require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
+require 'pub_sub_model_sync/run_subscriber'
 
 require 'pub_sub_model_sync/publisher'
 require 'pub_sub_model_sync/subscriber'

data/lib/pub_sub_model_sync/base.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Base
-    delegate :config, :log, to: self
+    delegate :config, :log, :debug?, to: self
 
     class << self
       def config
@@ -12,13 +12,30 @@ module PubSubModelSync
       def log(message, kind = :info)
         config.log message, kind
       end
+
+      def debug?
+        config.debug
+      end
     end
 
-    def retry_error(error_klass, qty: 2, &block)
-      @retries ||= 0
+    # @param errors (Array(Class|String))
+    def retry_error(errors, qty: 2, &block)
+      retries ||= 0
       block.call
-    rescue error_klass => _e
-      (@retries += 1) <= qty ? retry : raise
+    rescue => e
+      retries += 1
+      res = errors.find { |e_type| match_error?(e, e_type) }
+      raise if !res || retries > qty
+
+      sleep(qty * 0.1) && retry
+    end
+
+    private
+
+    # @param error (Exception)
+    # @param error_type (Class|String)
+    def match_error?(error, error_type)
+      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
     end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -3,12 +3,12 @@
 module PubSubModelSync
   class Config
     cattr_accessor(:subscribers) { [] }
-    cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
 
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
+    cattr_accessor(:transactions_use_buffer) { true }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -16,16 +16,15 @@ module PubSubModelSync
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
     cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
-    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
-    cattr_accessor :project, :credentials, :topic_name, :subscription_name
+    cattr_accessor :project, :credentials, :topic_name, :subscription_name, :default_topic_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name, :default_topic_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name, :subscription_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name, :default_topic_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
@@ -37,8 +36,17 @@ module PubSubModelSync
     end
 
     def self.subscription_key
-      subscription_name ||
-        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+      klass = Rails.application.class
+      app_name = klass.respond_to?(:module_parent_name) ? klass.module_parent_name : klass.parent_name
+      subscription_name || app_name
+    end
+
+    class << self
+      alias default_topic_name_old default_topic_name
+
+      def default_topic_name
+        default_topic_name_old || Array(topic_name).first
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload, :raise_error
+    attr_accessor :payload
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -15,22 +15,28 @@ module PubSubModelSync
       @payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
     end
 
-    def process
+    def process!
       filter_subscribers.each(&method(:run_subscriber))
     end
 
+    def process
+      process!
+    rescue => e
+      notify_error(e)
+    end
+
     private
 
     def run_subscriber(subscriber)
+      processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
-        subscriber.process!(payload)
+      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
+      retry_error(errors, qty: 5) do
+        processor.call
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log
       end
-    rescue => e
-      raise_error ? raise : print_subscriber_error(e, subscriber)
     end
 
     def processable?(subscriber)
@@ -40,16 +46,15 @@ module PubSubModelSync
     end
 
     # @param error (Error)
-    def print_subscriber_error(error, subscriber)
+    def notify_error(error)
       info = [payload, error.message, error.backtrace]
-      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
+      res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 
     def filter_subscribers
       config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
-          subscriber.settings[:from_action].to_s == payload.action.to_s
+        subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
       end
     end
   end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -3,45 +3,117 @@
 module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
+      class MissingPublisher < StandardError; end
+      attr_accessor :current_transaction
+
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
-      def publish_data(klass, data, action)
-        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
-        publish(payload)
+      # Permits to group all payloads with the same ordering_key and be processed in the same order
+      #   they are published by the subscribers. Grouping by ordering_key allows us to enable
+      #   multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
+      #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
+      #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
+      #   worker/thread.
+      # @see Transaction.new(...)
+      # @param key (String|Nil)
+      # @param block (Yield) block to be executed
+      def transaction(key, settings = {}, &block)
+        t = init_transaction(key, settings)
+        block.call
+        t.deliver_all
+      rescue
+        t.rollback
+        raise
+      ensure
+        t.clean_publisher
       end
 
-      # @param model: ActiveRecord model
-      # @param action: (Sym) Action name
-      # @param publisher: (Publisher, optional) Publisher to be used
-      def publish_model(model, action, publisher = nil)
-        return if model.ps_skip_sync?(action)
-
-        publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload.data)
-        return if res_before == :cancel
+      # Starts a new transaction
+      # @param key (@transaction_key)
+      # @return (Transaction)
+      def init_transaction(key, settings = {})
+        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        if current_transaction
+          current_transaction.add_transaction(new_transaction)
+        else
+          self.current_transaction = new_transaction
+        end
+        new_transaction
+      end
 
+      # Publishes a class level notification via pubsub
+      # @refer PublisherConcern.ps_class_publish
+      # @return Payload
+      def publish_data(klass, data, action, headers: {})
+        attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
+        payload = PubSubModelSync::Payload.new(data, attrs, headers)
         publish(payload)
-        model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload, raise_error: false)
-        if config.on_before_publish.call(payload) == :cancel
-          log("Publish message cancelled: #{payload}") if config.debug
-          return
+      # @param model (ActiveRecord::Base)
+      # @param action (Symbol: @see PublishConcern::ps_publish)
+      # @param settings (Hash: @see Publisher.new.settings)
+      def publish_model(model, action, settings = {})
+        return if model.ps_skip_publish?(action)
+
+        publisher = PubSubModelSync::Publisher.new(model, action, settings)
+        payload = publisher.payload
+
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
+          publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
         end
+      end
 
-        log("Publishing message: #{[payload]}")
+      # Publishes payload to pubsub
+      # @param payload (PubSubModelSync::Payload)
+      # @return Payload
+      # Raises error if exist
+      def publish!(payload, &block)
+        payload.headers[:ordering_key] = ordering_key_for(payload)
+        return unless ensure_publish(payload)
+
+        current_transaction ? current_transaction.add_payload(payload) : connector_publish(payload)
+        block&.call
+        payload
+      end
+
+      def connector_publish(payload)
         connector.publish(payload)
+        log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)
+      end
+
+      # Similar to :publish! method
+      # Notifies error via :on_error_publish instead of raising error
+      # @return Payload
+      def publish(payload, &block)
+        publish!(payload, &block)
       rescue => e
-        raise_error ? raise : notify_error(e, payload)
+        notify_error(e, payload)
       end
 
       private
 
+      def ensure_publish(payload)
+        cancelled = config.on_before_publish.call(payload) == :cancel
+        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
+      def ordering_key_for(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+        payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
+      end
+
+      def ensure_model_publish(model, action, payload)
+        res_before = model.ps_before_publish(action, payload)
+        cancelled = res_before == :cancel
+        log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
         res = config.on_error_publish.call(exception, { payload: payload })

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -26,6 +26,10 @@ module PubSubModelSync
     end
 
     class MockTopic
+      def name
+        'name'
+      end
+
       def subscription(*_args)
         @subscription ||= MockSubscription.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -28,16 +28,29 @@ module PubSubModelSync
       def subscribe(*_args)
         true
       end
+
+      def mark_message_as_processed(*_args)
+        true
+      end
     end
 
     def producer(*_args)
       MockProducer.new
     end
+    alias async_producer producer
 
     def consumer(*_args)
       MockConsumer.new
     end
 
+    def topics
+      []
+    end
+
+    def create_topic(_name)
+      true
+    end
+
     def close
       true
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -3,13 +3,25 @@
 module PubSubModelSync
   class Payload
     class MissingInfo < StandardError; end
-    attr_reader :data, :attributes, :headers
+    attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass: string, action: :sym }):
-    def initialize(data, attributes, headers = {})
+    # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
+    # @param headers (Hash):
+    #   key (String): identifier of the payload, default:
+    #        klass/action: when class message
+    #        klass/action/model.id: when model message
+    #   ordering_key (String): messages with the same key are processed in the same order they
+    #     were delivered, default:
+    #        klass: when class message
+    #        klass/id: when model message
+    #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
+    #     message (default first topic)
+    #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
+    #     even withing transactions. Default nil.
+    def initialize(data, info, headers = {})
       @data = data
-      @attributes = attributes
+      @info = { mode: :model }.merge(info)
       @headers = headers
       build_headers
       validate!
@@ -17,30 +29,32 @@ module PubSubModelSync
 
     # @return Hash: payload data
     def to_h
-      { data: data, attributes: attributes, headers: headers }
+      { data: data, info: info, headers: headers }
     end
 
     def klass
-      attributes[:klass]
+      info[:klass].to_s
     end
 
     def action
-      attributes[:action]
+      info[:action].to_sym
+    end
+
+    def mode
+      info[:mode].to_sym
     end
 
     # Process payload data
     #   (If error will raise exception and wont call on_error_processing callback)
     def process!
-      process do |publisher|
-        publisher.raise_error = true
-      end
+      publisher = PubSubModelSync::MessageProcessor.new(self)
+      publisher.process!
     end
 
     # Process payload data
     #   (If error will call on_error_processing callback)
     def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
-      yield(publisher) if block_given?
       publisher.process
     end
 
@@ -48,7 +62,7 @@ module PubSubModelSync
     #   (If error will raise exception and wont call on_error_publish callback)
     def publish!
       klass = PubSubModelSync::MessagePublisher
-      klass.publish(self, raise_error: true)
+      klass.publish!(self)
     end
 
     # Publish payload to pubsub
@@ -59,21 +73,23 @@ module PubSubModelSync
     end
 
     # convert payload data into Payload
-    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
       data = data.deep_symbolize_keys
-      new(data[:data], data[:attributes], data[:headers])
+      new(data[:data], data[:info], data[:headers])
     end
 
     private
 
     def build_headers
-      headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+      headers[:key] ||= [klass, action].join('/')
+      headers[:ordering_key] ||= klass
+      headers[:uuid] ||= SecureRandom.uuid
     end
 
     def validate!
-      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+      raise MissingInfo if !info[:klass] || !info[:action]
     end
   end
 end