pub_sub_model_sync 0.5.10 → 0.6.0

data/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: sh -c 'bundle exec rspec'
+    volumes:
+      - .:/myapp
+      - bundler_gems:/usr/local/bundle/
+
+volumes:
+  bundler_gems:

data/lib/pub_sub_model_sync/base.rb

@@ -14,11 +14,24 @@ module PubSubModelSync
       end
     end
 
-    def retry_error(error_klass, qty: 2, &block)
+    # @param errors (Array(Class|String))
+    def retry_error(errors, qty: 2, &block)
       retries ||= 0
       block.call
-    rescue error_klass => _e
-      (retries += 1) <= qty ? retry : raise
+    rescue => e
+      retries += 1
+      res = errors.find { |e_type| match_error?(e, e_type) }
+      raise if !res || retries > qty
+
+      sleep(qty * 0.1) && retry
+    end
+
+    private
+
+    # @param error (Exception)
+    # @param error_type (Class|String)
+    def match_error?(error, error_type)
+      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
     end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -22,7 +22,7 @@ module PubSubModelSync
     cattr_accessor :project, :credentials, :topic_name, :subscription_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name
 
     # kafka service
     cattr_accessor :kafka_connection, :topic_name, :subscription_name

data/lib/pub_sub_model_sync/message_processor.rb

@@ -28,9 +28,11 @@ module PubSubModelSync
     private
 
     def run_subscriber(subscriber)
+      subscriber = subscriber.dup
       return unless processable?(subscriber)
 
-      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
+      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
+      retry_error(errors, qty: 2) do
         subscriber.process!(payload)
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -3,60 +3,127 @@
 module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
+      class MissingPublisher < StandardError; end
+      attr_accessor :transaction_key
+
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
+      # Permits to group all payloads with the same ordering_key and be processed in the same order
+      #   they are published by the subscribers. Grouping by ordering_key allows us to enable
+      #   multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
+      #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
+      #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
+      #   worker/thread.
+      # @param key (String): This key will be used as the ordering_key for all payload
+      #     inside this transaction.
+      def transaction(key, &block)
+        parent_key = init_transaction(key)
+        begin
+          block.call
+        ensure
+          end_transaction(parent_key)
+        end
+      end
+
+      # Starts a new transaction
+      # @return (String) returns parent transaction key
+      def init_transaction(key)
+        parent_key = transaction_key
+        self.transaction_key = transaction_key.presence || key
+        parent_key
+      end
+
+      # Restores to the last transaction key
+      def end_transaction(parent_key)
+        self.transaction_key = parent_key
+      end
+
       # Publishes any value to pubsub
       # @param klass (String): Class name
       # @param data (Hash): Data to be delivered
       # @param action (:symbol): action name
-      def publish_data(klass, data, action)
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      # @return Payload
+      def publish_data(klass, data, action, headers: {})
         attrs = { klass: klass.to_s, action: action.to_sym }
-        payload = PubSubModelSync::Payload.new(data, attrs)
+        payload = PubSubModelSync::Payload.new(data, attrs, headers)
         publish(payload)
       end
 
+      # Publishes custom model action
+      # @param model (ActiveRecord): Model object owner of the data
+      # @param data (Hash): Data to be delivered
+      # @param action (:symbol): action name
+      # @param as_klass (String, optional): Class name (default model class name)
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      # @return Payload
+      def publish_model_data(model, data, action, as_klass: nil, headers: {})
+        headers = PubSubModelSync::Publisher.headers_for(model, action).merge(headers)
+        publish_data(as_klass || model.class.name, data, action, headers: headers)
+      end
+
       # Publishes model info to pubsub
       # @param model (ActiveRecord model)
       # @param action (Sym): Action name
-      # @param publisher (Publisher, optional): Publisher to be used
-      def publish_model(model, action, publisher = nil)
+      # @param custom_data (Hash, optional): If present custom_data will be used as the payload data.
+      # @param custom_headers (Hash): Refer Payload.headers
+      # @return Payload
+      def publish_model(model, action, custom_data: nil, custom_headers: {})
         return if model.ps_skip_sync?(action)
 
-        publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload.data)
-        return if res_before == :cancel
+        publisher = model.class.ps_publisher(action)
+        error_msg = "No publisher found for: \"#{[model.class.name, action]}\""
+        raise(MissingPublisher, error_msg) unless publisher
 
-        publish(payload)
-        model.ps_after_sync(action, payload.data)
+        payload = publisher.payload(model, action, custom_data: custom_data, custom_headers: custom_headers)
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_sync syncs
+          publish(payload) { model.ps_after_sync(action, payload) } if ensure_model_publish(model, action, payload)
+        end
       end
 
       # Publishes payload to pubsub
-      # @attr payload (PubSubModelSync::Payload)
+      # @param payload (PubSubModelSync::Payload)
+      # @return Payload
       # Raises error if exist
-      def publish!(payload)
-        if config.on_before_publish.call(payload) == :cancel
-          log("Publish message cancelled: #{payload}") if config.debug
-          return
-        end
+      def publish!(payload, &block)
+        return unless ensure_publish(payload)
 
         log("Publishing message: #{[payload]}")
         connector.publish(payload)
         config.on_after_publish.call(payload)
+        block&.call
+        payload
       end
 
       # Similar to :publish! method
       # Notifies error via :on_error_publish instead of raising error
-      def publish(payload)
-        publish!(payload)
+      # @return Payload
+      def publish(payload, &block)
+        publish!(payload, &block)
       rescue => e
         notify_error(e, payload)
       end
 
       private
 
+      def ensure_publish(payload)
+        payload.headers[:ordering_key] = @transaction_key if @transaction_key.present?
+        forced_ordering_key = payload.headers[:forced_ordering_key]
+        payload.headers[:ordering_key] = forced_ordering_key if forced_ordering_key
+        cancelled = config.on_before_publish.call(payload) == :cancel
+        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
+      def ensure_model_publish(model, action, payload)
+        res_before = model.ps_before_sync(action, payload)
+        cancelled = res_before == :cancel
+        log("Publish cancelled by model.ps_before_sync: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
         res = config.on_error_publish.call(exception, { payload: payload })

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -26,6 +26,10 @@ module PubSubModelSync
     end
 
     class MockTopic
+      def name
+        'name'
+      end
+
       def subscription(*_args)
         @subscription ||= MockSubscription.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -28,16 +28,29 @@ module PubSubModelSync
       def subscribe(*_args)
         true
       end
+
+      def mark_message_as_processed(*_args)
+        true
+      end
     end
 
     def producer(*_args)
       MockProducer.new
     end
+    alias async_producer producer
 
     def consumer(*_args)
       MockConsumer.new
     end
 
+    def topics
+      []
+    end
+
+    def create_topic(_name)
+      true
+    end
+
     def close
       true
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -7,7 +7,18 @@ module PubSubModelSync
 
     # @param data (Hash: { any value }):
     # @param attributes (Hash: { klass*: string, action*: :sym }):
-    # @param headers (Hash: { key?: string, ...any_key?: anything }):
+    # @param headers (Hash):
+    #   key (String): identifier of the payload, default:
+    #        klass/action: when class message
+    #        klass/action/model.id: when model message
+    #   ordering_key (String): messages with the same key are processed in the same order they
+    #     were delivered, default:
+    #        klass: when class message
+    #        klass/id: when model message
+    #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
+    #     message (default first topic)
+    #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
+    #     even withing transactions. Default nil.
     def initialize(data, attributes, headers = {})
       @data = data
       @attributes = attributes
@@ -22,7 +33,7 @@ module PubSubModelSync
     end
 
     def klass
-      attributes[:klass]
+      attributes[:klass].to_s
     end
 
     def action
@@ -67,9 +78,10 @@ module PubSubModelSync
     private
 
     def build_headers
-      headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
-      headers[:key] ||= [klass.to_s, action].join('/')
+      headers[:key] ||= [klass, action].join('/')
+      headers[:ordering_key] ||= klass
+      headers[:uuid] ||= SecureRandom.uuid
     end
 
     def validate!

data/lib/pub_sub_model_sync/publisher.rb

@@ -2,32 +2,43 @@
 
 module PubSubModelSync
   class Publisher
-    attr_accessor :attrs, :actions, :klass, :as_klass
+    attr_accessor :attrs, :actions, :klass, :as_klass, :headers
 
-    def initialize(attrs, klass, actions = nil, as_klass = nil)
+    # @param headers (Hash): refer Payload.headers
+    def initialize(attrs, klass, actions = nil, as_klass: nil, headers: {})
       @attrs = attrs
       @klass = klass
       @actions = actions || %i[create update destroy]
       @as_klass = as_klass || klass
+      @headers = headers
     end
 
     # Builds the payload with model information defined for :action (:create|:update|:destroy)
-    def payload(model, action)
-      headers = { key: [model.class.name, action, model.id].join('/') }
-      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action), headers)
+    # @param custom_headers (Hash, default {}): refer Payload.headers
+    def payload(model, action, custom_data: nil, custom_headers: {})
+      payload_headers = self.class.headers_for(model, action)
+                            .merge(headers)
+                            .merge(custom_headers)
+      data = custom_data || payload_data(model)
+      PubSubModelSync::Payload.new(data, payload_attrs(model, action), payload_headers)
+    end
+
+    def self.headers_for(model, action)
+      key = [model.class.name, action, model.id].join('/')
+      { ordering_key: ordering_key_for(model), key: key }
+    end
+
+    def self.ordering_key_for(model)
+      [model.class.name, model.id || SecureRandom.uuid].join('/')
     end
 
     private
 
     def payload_data(model)
-      source_props = @attrs.map { |prop| prop.to_s.split(':').first }
-      data = model.as_json(only: source_props, methods: source_props)
-      aliased_props = @attrs.select { |prop| prop.to_s.include?(':') }
-      aliased_props.each do |prop|
+      @attrs.map do |prop|
         source, target = prop.to_s.split(':')
-        data[target] = data.delete(source)
-      end
-      data.symbolize_keys
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
     end
 
     def payload_attrs(model, action)

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -4,6 +4,7 @@ module PubSubModelSync
   module PublisherConcern
     def self.included(base)
       base.extend(ClassMethods)
+      base.send(:ps_init_transaction_callbacks)
     end
 
     # Before initializing sync service (callbacks: after create/update/destroy)
@@ -17,39 +18,39 @@ module PubSubModelSync
     end
 
     # before delivering data (return :cancel to cancel sync)
-    def ps_before_sync(_action, _data); end
+    def ps_before_sync(_action, _payload); end
 
     # after delivering data
-    def ps_after_sync(_action, _data); end
+    def ps_after_sync(_action, _payload); end
 
     # To perform sync on demand
-    # @param attrs (Array, optional): custom attrs to be used
-    # @param as_klass (Array, optional): custom klass name to be used
-    # @param publisher (Publisher, optional): custom publisher object
-    def ps_perform_sync(action = :create, attrs: nil, as_klass: nil,
-                        publisher: nil)
-      publisher ||= self.class.ps_publisher(action).dup
-      publisher.attrs = attrs if attrs
-      publisher.as_klass = as_klass if as_klass
-      PubSubModelSync::MessagePublisher.publish_model(self, action, publisher)
+    # @param action (Sym): CRUD action name
+    # @param custom_data (nil|Hash) If present custom_data will be used as the payload data. I.E.
+    #   data generator will be ignored
+    # @param custom_headers (Hash, optional): refer Payload.headers
+    def ps_perform_sync(action = :create, custom_data: nil, custom_headers: {})
+      p_klass = PubSubModelSync::MessagePublisher
+      p_klass.publish_model(self, action, custom_data: custom_data, custom_headers: custom_headers)
     end
 
     module ClassMethods
       # Permit to configure to publish crud actions (:create, :update, :destroy)
-      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil)
+      # @param headers (Hash, optional): Refer Payload.headers
+      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil, headers: {})
         klass = PubSubModelSync::Publisher
-        publisher = klass.new(attrs, name, actions, as_klass)
+        publisher = klass.new(attrs, name, actions, as_klass: as_klass, headers: headers)
         PubSubModelSync::Config.publishers << publisher
         actions.each do |action|
-          ps_register_callback(action.to_sym, publisher)
+          ps_register_callback(action.to_sym)
         end
       end
 
-      # On demand class level publisher
-      def ps_class_publish(data, action:, as_klass: nil)
-        as_klass = (as_klass || name).to_s
+      # Klass level notification
+      # @deprecated this method was deprecated in favor of:
+      #   PubSubModelSync::MessagePublisher.publish_data(...)
+      def ps_class_publish(data, action:, as_klass: nil, headers: {})
         klass = PubSubModelSync::MessagePublisher
-        klass.publish_data(as_klass, data, action.to_sym)
+        klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
       end
 
       # Publisher info for specific action
@@ -61,12 +62,28 @@ module PubSubModelSync
 
       private
 
-      def ps_register_callback(action, publisher)
+      # TODO: skip all enqueued notifications after_rollback (when failed)
+      # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
+      def ps_init_transaction_callbacks
+        start_transaction = lambda do
+          key = PubSubModelSync::Publisher.ordering_key_for(self)
+          @ps_old_transaction_key = PubSubModelSync::MessagePublisher.init_transaction(key)
+        end
+        end_transaction = -> { PubSubModelSync::MessagePublisher.end_transaction(@ps_old_transaction_key) }
+        after_create start_transaction, prepend: true # wait for ID
+        before_update start_transaction, prepend: true
+        before_destroy start_transaction, prepend: true
+        after_commit end_transaction
+        after_rollback end_transaction
+      end
+
+      # Configure specific callback and execute publisher when called callback
+      def ps_register_callback(action)
         after_commit(on: action) do |model|
           disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
           if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
-            klass.publish_model(model, action.to_sym, publisher)
+            klass.publish_model(model, action.to_sym)
           end
         end
       end