pub_sub_model_sync 0.5.8.1 → 0.6.0

data/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: sh -c 'bundle exec rspec'
+    volumes:
+      - .:/myapp
+      - bundler_gems:/usr/local/bundle/
+
+volumes:
+  bundler_gems:

data/lib/pub_sub_model_sync/base.rb

@@ -14,11 +14,24 @@ module PubSubModelSync
       end
     end
 
-    def retry_error(error_klass, qty: 2, &block)
-      @retries ||= 0
+    # @param errors (Array(Class|String))
+    def retry_error(errors, qty: 2, &block)
+      retries ||= 0
       block.call
-    rescue error_klass => _e
-      (@retries += 1) <= qty ? retry : raise
+    rescue => e
+      retries += 1
+      res = errors.find { |e_type| match_error?(e, e_type) }
+      raise if !res || retries > qty
+
+      sleep(qty * 0.1) && retry
+    end
+
+    private
+
+    # @param error (Exception)
+    # @param error_type (Class|String)
+    def match_error?(error, error_type)
+      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
     end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -22,7 +22,7 @@ module PubSubModelSync
     cattr_accessor :project, :credentials, :topic_name, :subscription_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name
 
     # kafka service
     cattr_accessor :kafka_connection, :topic_name, :subscription_name

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload, :raise_error
+    attr_accessor :payload
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -15,22 +15,28 @@ module PubSubModelSync
       @payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
     end
 
-    def process
+    def process!
       filter_subscribers.each(&method(:run_subscriber))
     end
 
+    def process
+      process!
+    rescue => e
+      notify_error(e)
+    end
+
     private
 
     def run_subscriber(subscriber)
+      subscriber = subscriber.dup
       return unless processable?(subscriber)
 
-      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
+      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
+      retry_error(errors, qty: 2) do
         subscriber.process!(payload)
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log
       end
-    rescue => e
-      raise_error ? raise : print_subscriber_error(e, subscriber)
     end
 
     def processable?(subscriber)
@@ -40,9 +46,9 @@ module PubSubModelSync
     end
 
     # @param error (Error)
-    def print_subscriber_error(error, subscriber)
+    def notify_error(error)
       info = [payload, error.message, error.backtrace]
-      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
+      res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -3,45 +3,127 @@
 module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
+      class MissingPublisher < StandardError; end
+      attr_accessor :transaction_key
+
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
-      def publish_data(klass, data, action)
-        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
-        publish(payload)
+      # Permits to group all payloads with the same ordering_key and be processed in the same order
+      #   they are published by the subscribers. Grouping by ordering_key allows us to enable
+      #   multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
+      #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
+      #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
+      #   worker/thread.
+      # @param key (String): This key will be used as the ordering_key for all payload
+      #     inside this transaction.
+      def transaction(key, &block)
+        parent_key = init_transaction(key)
+        begin
+          block.call
+        ensure
+          end_transaction(parent_key)
+        end
       end
 
-      # @param model: ActiveRecord model
-      # @param action: (Sym) Action name
-      # @param publisher: (Publisher, optional) Publisher to be used
-      def publish_model(model, action, publisher = nil)
-        return if model.ps_skip_sync?(action)
+      # Starts a new transaction
+      # @return (String) returns parent transaction key
+      def init_transaction(key)
+        parent_key = transaction_key
+        self.transaction_key = transaction_key.presence || key
+        parent_key
+      end
 
-        publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload.data)
-        return if res_before == :cancel
+      # Restores to the last transaction key
+      def end_transaction(parent_key)
+        self.transaction_key = parent_key
+      end
 
+      # Publishes any value to pubsub
+      # @param klass (String): Class name
+      # @param data (Hash): Data to be delivered
+      # @param action (:symbol): action name
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      # @return Payload
+      def publish_data(klass, data, action, headers: {})
+        attrs = { klass: klass.to_s, action: action.to_sym }
+        payload = PubSubModelSync::Payload.new(data, attrs, headers)
         publish(payload)
-        model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload, raise_error: false)
-        if config.on_before_publish.call(payload) == :cancel
-          log("Publish message cancelled: #{payload}") if config.debug
-          return
+      # Publishes custom model action
+      # @param model (ActiveRecord): Model object owner of the data
+      # @param data (Hash): Data to be delivered
+      # @param action (:symbol): action name
+      # @param as_klass (String, optional): Class name (default model class name)
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      # @return Payload
+      def publish_model_data(model, data, action, as_klass: nil, headers: {})
+        headers = PubSubModelSync::Publisher.headers_for(model, action).merge(headers)
+        publish_data(as_klass || model.class.name, data, action, headers: headers)
+      end
+
+      # Publishes model info to pubsub
+      # @param model (ActiveRecord model)
+      # @param action (Sym): Action name
+      # @param custom_data (Hash, optional): If present custom_data will be used as the payload data.
+      # @param custom_headers (Hash): Refer Payload.headers
+      # @return Payload
+      def publish_model(model, action, custom_data: nil, custom_headers: {})
+        return if model.ps_skip_sync?(action)
+
+        publisher = model.class.ps_publisher(action)
+        error_msg = "No publisher found for: \"#{[model.class.name, action]}\""
+        raise(MissingPublisher, error_msg) unless publisher
+
+        payload = publisher.payload(model, action, custom_data: custom_data, custom_headers: custom_headers)
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_sync syncs
+          publish(payload) { model.ps_after_sync(action, payload) } if ensure_model_publish(model, action, payload)
         end
+      end
+
+      # Publishes payload to pubsub
+      # @param payload (PubSubModelSync::Payload)
+      # @return Payload
+      # Raises error if exist
+      def publish!(payload, &block)
+        return unless ensure_publish(payload)
 
         log("Publishing message: #{[payload]}")
         connector.publish(payload)
         config.on_after_publish.call(payload)
+        block&.call
+        payload
+      end
+
+      # Similar to :publish! method
+      # Notifies error via :on_error_publish instead of raising error
+      # @return Payload
+      def publish(payload, &block)
+        publish!(payload, &block)
       rescue => e
-        raise_error ? raise : notify_error(e, payload)
+        notify_error(e, payload)
       end
 
       private
 
+      def ensure_publish(payload)
+        payload.headers[:ordering_key] = @transaction_key if @transaction_key.present?
+        forced_ordering_key = payload.headers[:forced_ordering_key]
+        payload.headers[:ordering_key] = forced_ordering_key if forced_ordering_key
+        cancelled = config.on_before_publish.call(payload) == :cancel
+        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
+      def ensure_model_publish(model, action, payload)
+        res_before = model.ps_before_sync(action, payload)
+        cancelled = res_before == :cancel
+        log("Publish cancelled by model.ps_before_sync: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
         res = config.on_error_publish.call(exception, { payload: payload })

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -26,6 +26,10 @@ module PubSubModelSync
     end
 
     class MockTopic
+      def name
+        'name'
+      end
+
       def subscription(*_args)
         @subscription ||= MockSubscription.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -28,16 +28,29 @@ module PubSubModelSync
       def subscribe(*_args)
         true
       end
+
+      def mark_message_as_processed(*_args)
+        true
+      end
     end
 
     def producer(*_args)
       MockProducer.new
     end
+    alias async_producer producer
 
     def consumer(*_args)
       MockConsumer.new
     end
 
+    def topics
+      []
+    end
+
+    def create_topic(_name)
+      true
+    end
+
     def close
       true
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -6,7 +6,19 @@ module PubSubModelSync
     attr_reader :data, :attributes, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass: string, action: :sym }):
+    # @param attributes (Hash: { klass*: string, action*: :sym }):
+    # @param headers (Hash):
+    #   key (String): identifier of the payload, default:
+    #        klass/action: when class message
+    #        klass/action/model.id: when model message
+    #   ordering_key (String): messages with the same key are processed in the same order they
+    #     were delivered, default:
+    #        klass: when class message
+    #        klass/id: when model message
+    #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
+    #     message (default first topic)
+    #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
+    #     even withing transactions. Default nil.
     def initialize(data, attributes, headers = {})
       @data = data
       @attributes = attributes
@@ -21,7 +33,7 @@ module PubSubModelSync
     end
 
     def klass
-      attributes[:klass]
+      attributes[:klass].to_s
     end
 
     def action
@@ -31,16 +43,14 @@ module PubSubModelSync
     # Process payload data
     #   (If error will raise exception and wont call on_error_processing callback)
     def process!
-      process do |publisher|
-        publisher.raise_error = true
-      end
+      publisher = PubSubModelSync::MessageProcessor.new(self)
+      publisher.process!
     end
 
     # Process payload data
     #   (If error will call on_error_processing callback)
     def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
-      yield(publisher) if block_given?
       publisher.process
     end
 
@@ -48,7 +58,7 @@ module PubSubModelSync
     #   (If error will raise exception and wont call on_error_publish callback)
     def publish!
       klass = PubSubModelSync::MessagePublisher
-      klass.publish(self, raise_error: true)
+      klass.publish!(self)
     end
 
     # Publish payload to pubsub
@@ -68,8 +78,10 @@ module PubSubModelSync
     private
 
     def build_headers
-      headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+      headers[:key] ||= [klass, action].join('/')
+      headers[:ordering_key] ||= klass
+      headers[:uuid] ||= SecureRandom.uuid
     end
 
     def validate!

data/lib/pub_sub_model_sync/publisher.rb

@@ -2,34 +2,50 @@
 
 module PubSubModelSync
   class Publisher
-    attr_accessor :attrs, :actions, :klass, :as_klass
+    attr_accessor :attrs, :actions, :klass, :as_klass, :headers
 
-    def initialize(attrs, klass, actions = nil, as_klass = nil)
+    # @param headers (Hash): refer Payload.headers
+    def initialize(attrs, klass, actions = nil, as_klass: nil, headers: {})
       @attrs = attrs
       @klass = klass
       @actions = actions || %i[create update destroy]
       @as_klass = as_klass || klass
+      @headers = headers
     end
 
-    def payload(model, action)
-      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
+    # Builds the payload with model information defined for :action (:create|:update|:destroy)
+    # @param custom_headers (Hash, default {}): refer Payload.headers
+    def payload(model, action, custom_data: nil, custom_headers: {})
+      payload_headers = self.class.headers_for(model, action)
+                            .merge(headers)
+                            .merge(custom_headers)
+      data = custom_data || payload_data(model)
+      PubSubModelSync::Payload.new(data, payload_attrs(model, action), payload_headers)
+    end
+
+    def self.headers_for(model, action)
+      key = [model.class.name, action, model.id].join('/')
+      { ordering_key: ordering_key_for(model), key: key }
+    end
+
+    def self.ordering_key_for(model)
+      [model.class.name, model.id || SecureRandom.uuid].join('/')
     end
 
     private
 
     def payload_data(model)
-      source_props = @attrs.map { |prop| prop.to_s.split(':').first }
-      data = model.as_json(only: source_props, methods: source_props)
-      aliased_props = @attrs.select { |prop| prop.to_s.include?(':') }
-      aliased_props.each do |prop|
+      @attrs.map do |prop|
         source, target = prop.to_s.split(':')
-        data[target] = data.delete(source)
-      end
-      data.symbolize_keys
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
     end
 
     def payload_attrs(model, action)
-      { klass: (as_klass || model.class.name).to_s, action: action.to_sym }
+      {
+        klass: (as_klass || model.class.name).to_s,
+        action: action.to_sym
+      }
     end
   end
 end

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -4,6 +4,7 @@ module PubSubModelSync
   module PublisherConcern
     def self.included(base)
       base.extend(ClassMethods)
+      base.send(:ps_init_transaction_callbacks)
     end
 
     # Before initializing sync service (callbacks: after create/update/destroy)
@@ -17,39 +18,39 @@ module PubSubModelSync
     end
 
     # before delivering data (return :cancel to cancel sync)
-    def ps_before_sync(_action, _data); end
+    def ps_before_sync(_action, _payload); end
 
     # after delivering data
-    def ps_after_sync(_action, _data); end
+    def ps_after_sync(_action, _payload); end
 
     # To perform sync on demand
-    # @param attrs (Array, optional): custom attrs to be used
-    # @param as_klass (Array, optional): custom klass name to be used
-    # @param publisher (Publisher, optional): custom publisher object
-    def ps_perform_sync(action = :create, attrs: nil, as_klass: nil,
-                        publisher: nil)
-      publisher ||= self.class.ps_publisher(action).dup
-      publisher.attrs = attrs if attrs
-      publisher.as_klass = as_klass if as_klass
-      PubSubModelSync::MessagePublisher.publish_model(self, action, publisher)
+    # @param action (Sym): CRUD action name
+    # @param custom_data (nil|Hash) If present custom_data will be used as the payload data. I.E.
+    #   data generator will be ignored
+    # @param custom_headers (Hash, optional): refer Payload.headers
+    def ps_perform_sync(action = :create, custom_data: nil, custom_headers: {})
+      p_klass = PubSubModelSync::MessagePublisher
+      p_klass.publish_model(self, action, custom_data: custom_data, custom_headers: custom_headers)
     end
 
     module ClassMethods
       # Permit to configure to publish crud actions (:create, :update, :destroy)
-      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil)
+      # @param headers (Hash, optional): Refer Payload.headers
+      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil, headers: {})
         klass = PubSubModelSync::Publisher
-        publisher = klass.new(attrs, name, actions, as_klass)
+        publisher = klass.new(attrs, name, actions, as_klass: as_klass, headers: headers)
         PubSubModelSync::Config.publishers << publisher
         actions.each do |action|
-          ps_register_callback(action.to_sym, publisher)
+          ps_register_callback(action.to_sym)
         end
       end
 
-      # On demand class level publisher
-      def ps_class_publish(data, action:, as_klass: nil)
-        as_klass = (as_klass || name).to_s
+      # Klass level notification
+      # @deprecated this method was deprecated in favor of:
+      #   PubSubModelSync::MessagePublisher.publish_data(...)
+      def ps_class_publish(data, action:, as_klass: nil, headers: {})
         klass = PubSubModelSync::MessagePublisher
-        klass.publish_data(as_klass, data, action.to_sym)
+        klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
       end
 
       # Publisher info for specific action
@@ -61,12 +62,28 @@ module PubSubModelSync
 
       private
 
-      def ps_register_callback(action, publisher)
+      # TODO: skip all enqueued notifications after_rollback (when failed)
+      # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
+      def ps_init_transaction_callbacks
+        start_transaction = lambda do
+          key = PubSubModelSync::Publisher.ordering_key_for(self)
+          @ps_old_transaction_key = PubSubModelSync::MessagePublisher.init_transaction(key)
+        end
+        end_transaction = -> { PubSubModelSync::MessagePublisher.end_transaction(@ps_old_transaction_key) }
+        after_create start_transaction, prepend: true # wait for ID
+        before_update start_transaction, prepend: true
+        before_destroy start_transaction, prepend: true
+        after_commit end_transaction
+        after_rollback end_transaction
+      end
+
+      # Configure specific callback and execute publisher when called callback
+      def ps_register_callback(action)
         after_commit(on: action) do |model|
           disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
           if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
-            klass.publish_model(model, action.to_sym, publisher)
+            klass.publish_model(model, action.to_sym)
           end
         end
       end