pub_sub_model_sync 0.5.9 → 1.0.beta1

data/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: sh -c 'bundle exec rspec'
+    volumes:
+      - .:/app
+      - bundler_gems:/usr/local/bundle/
+
+volumes:
+  bundler_gems:

data/lib/pub_sub_model_sync.rb

@@ -10,8 +10,10 @@ require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'
 require 'pub_sub_model_sync/runner'
+require 'pub_sub_model_sync/transaction'
 require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
+require 'pub_sub_model_sync/run_subscriber'
 
 require 'pub_sub_model_sync/publisher'
 require 'pub_sub_model_sync/subscriber'

data/lib/pub_sub_model_sync/base.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Base
-    delegate :config, :log, to: self
+    delegate :config, :log, :debug?, to: self
 
     class << self
       def config
@@ -12,13 +12,30 @@ module PubSubModelSync
       def log(message, kind = :info)
         config.log message, kind
       end
+
+      def debug?
+        config.debug
+      end
     end
 
-    def retry_error(error_klass, qty: 2, &block)
-      @retries ||= 0
+    # @param errors (Array(Class|String))
+    def retry_error(errors, qty: 2, &block)
+      retries ||= 0
       block.call
-    rescue error_klass => _e
-      (@retries += 1) <= qty ? retry : raise
+    rescue => e
+      retries += 1
+      res = errors.find { |e_type| match_error?(e, e_type) }
+      raise if !res || retries > qty
+
+      sleep(qty * 0.1) && retry
+    end
+
+    private
+
+    # @param error (Exception)
+    # @param error_type (Class|String)
+    def match_error?(error, error_type)
+      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
     end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -3,12 +3,13 @@
 module PubSubModelSync
   class Config
     cattr_accessor(:subscribers) { [] }
-    cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
 
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
+    cattr_accessor(:transactions_max_buffer) { 100 }
+    cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -16,16 +17,15 @@ module PubSubModelSync
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
     cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
-    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
-    cattr_accessor :project, :credentials, :topic_name, :subscription_name
+    cattr_accessor :project, :credentials, :topic_name, :subscription_name, :default_topic_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name, :default_topic_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name, :subscription_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name, :default_topic_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
@@ -37,8 +37,17 @@ module PubSubModelSync
     end
 
     def self.subscription_key
-      subscription_name ||
-        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+      klass = Rails.application.class
+      app_name = klass.respond_to?(:module_parent_name) ? klass.module_parent_name : klass.parent_name
+      subscription_name || app_name
+    end
+
+    class << self
+      alias default_topic_name_old default_topic_name
+
+      def default_topic_name
+        default_topic_name_old || Array(topic_name).first
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/initializers/before_commit.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
+ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
+  alias_method :commit_with_before_commit, :commit
+
+  def commit
+    call_before_commit_records if Rails::VERSION::MAJOR == 4
+    commit_with_before_commit
+  end
+
+  private
+
+  def call_before_commit_records
+    ite = records.uniq
+    ite.each do |record|
+      action = record.previous_changes.include?(:id) ? :create : :update
+      action = :destroy if record.destroyed?
+      callback_name = "ps_before_#{action}_commit".to_sym
+      record.send(callback_name) if record.respond_to?(callback_name)
+    end
+  end
+end

data/lib/pub_sub_model_sync/message_processor.rb

@@ -28,10 +28,12 @@ module PubSubModelSync
     private
 
     def run_subscriber(subscriber)
+      processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
-        subscriber.process!(payload)
+      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
+      retry_error(errors, qty: 5) do
+        processor.call
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log
       end
@@ -52,8 +54,7 @@ module PubSubModelSync
 
     def filter_subscribers
       config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
-          subscriber.settings[:from_action].to_s == payload.action.to_s
+        subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
       end
     end
   end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -3,60 +3,117 @@
 module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
+      class MissingPublisher < StandardError; end
+      attr_accessor :current_transaction
+
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
-      # Publishes any value to pubsub
-      # @param klass (String): Class name
-      # @param data (Hash): Data to be delivered
-      # @param action (:symbol): action name
-      def publish_data(klass, data, action)
-        attrs = { klass: klass.to_s, action: action.to_sym, key: [klass.to_s, action].join('/') }
-        payload = PubSubModelSync::Payload.new(data, attrs)
+      # Permits to group all payloads with the same ordering_key and be processed in the same order
+      #   they are published by the subscribers. Grouping by ordering_key allows us to enable
+      #   multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
+      #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
+      #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
+      #   worker/thread.
+      # @see Transaction.new(...)
+      # @param key (String|Nil)
+      # @param block (Yield) block to be executed
+      def transaction(key, settings = {}, &block)
+        t = init_transaction(key, settings)
+        block.call
+        t.finish
+      rescue
+        t.rollback
+        raise
+      ensure
+        t.clean_publisher
+      end
+
+      # Starts a new transaction
+      # @param key (@transaction_key)
+      # @return (Transaction)
+      def init_transaction(key, settings = {})
+        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        if current_transaction
+          current_transaction.add_transaction(new_transaction)
+        else
+          self.current_transaction = new_transaction
+        end
+        new_transaction
+      end
+
+      # Publishes a class level notification via pubsub
+      # @refer PublisherConcern.ps_class_publish
+      # @return Payload
+      def publish_data(klass, data, action, headers: {})
+        attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
+        payload = PubSubModelSync::Payload.new(data, attrs, headers)
         publish(payload)
       end
 
-      # Publishes model info to pubsub
-      # @param model (ActiveRecord model)
-      # @param action (Sym): Action name
-      # @param publisher (Publisher, optional): Publisher to be used
-      def publish_model(model, action, publisher = nil)
-        return if model.ps_skip_sync?(action)
+      # @param model (ActiveRecord::Base)
+      # @param action (Symbol: @see PublishConcern::ps_publish)
+      # @param settings (Hash: @see Publisher.new.settings)
+      def publish_model(model, action, settings = {})
+        return if model.ps_skip_publish?(action)
 
-        publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload.data)
-        return if res_before == :cancel
+        publisher = PubSubModelSync::Publisher.new(model, action, settings)
+        payload = publisher.payload
 
-        publish(payload)
-        model.ps_after_sync(action, payload.data)
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
+          publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
+        end
       end
 
       # Publishes payload to pubsub
-      # @attr payload (PubSubModelSync::Payload)
+      # @param payload (PubSubModelSync::Payload)
+      # @return Payload
       # Raises error if exist
-      def publish!(payload)
-        if config.on_before_publish.call(payload) == :cancel
-          log("Publish message cancelled: #{payload}") if config.debug
-          return
-        end
+      def publish!(payload, &block)
+        payload.headers[:ordering_key] = ordering_key_for(payload)
+        return unless ensure_publish(payload)
 
-        log("Publishing message: #{[payload]}")
+        current_transaction ? current_transaction.add_payload(payload) : connector_publish(payload)
+        block&.call
+        payload
+      end
+
+      def connector_publish(payload)
         connector.publish(payload)
+        log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)
       end
 
       # Similar to :publish! method
       # Notifies error via :on_error_publish instead of raising error
-      def publish(payload)
-        publish!(payload)
+      # @return Payload
+      def publish(payload, &block)
+        publish!(payload, &block)
       rescue => e
         notify_error(e, payload)
       end
 
       private
 
+      def ensure_publish(payload)
+        cancelled = config.on_before_publish.call(payload) == :cancel
+        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
+      def ordering_key_for(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+        payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
+      end
+
+      def ensure_model_publish(model, action, payload)
+        res_before = model.ps_before_publish(action, payload)
+        cancelled = res_before == :cancel
+        log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
         res = config.on_error_publish.call(exception, { payload: payload })

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -26,6 +26,10 @@ module PubSubModelSync
     end
 
     class MockTopic
+      def name
+        'name'
+      end
+
       def subscription(*_args)
         @subscription ||= MockSubscription.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -28,16 +28,29 @@ module PubSubModelSync
       def subscribe(*_args)
         true
       end
+
+      def mark_message_as_processed(*_args)
+        true
+      end
     end
 
     def producer(*_args)
       MockProducer.new
     end
+    alias async_producer producer
 
     def consumer(*_args)
       MockConsumer.new
     end
 
+    def topics
+      []
+    end
+
+    def create_topic(_name)
+      true
+    end
+
     def close
       true
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -3,13 +3,25 @@
 module PubSubModelSync
   class Payload
     class MissingInfo < StandardError; end
-    attr_reader :data, :attributes, :headers
+    attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass*: string, action*: :sym, key?: string }):
-    def initialize(data, attributes, headers = {})
+    # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
+    # @param headers (Hash):
+    #   key (String): identifier of the payload, default:
+    #        klass/action: when class message
+    #        klass/action/model.id: when model message
+    #   ordering_key (String): messages with the same key are processed in the same order they
+    #     were delivered, default:
+    #        klass: when class message
+    #        klass/id: when model message
+    #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
+    #     message (default first topic)
+    #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
+    #     even withing transactions. Default nil.
+    def initialize(data, info, headers = {})
       @data = data
-      @attributes = attributes
+      @info = { mode: :model }.merge(info)
       @headers = headers
       build_headers
       validate!
@@ -17,15 +29,19 @@ module PubSubModelSync
 
     # @return Hash: payload data
     def to_h
-      { data: data, attributes: attributes, headers: headers }
+      { data: data, info: info, headers: headers }
     end
 
     def klass
-      attributes[:klass]
+      info[:klass].to_s
     end
 
     def action
-      attributes[:action]
+      info[:action].to_sym
+    end
+
+    def mode
+      info[:mode].to_sym
     end
 
     # Process payload data
@@ -57,21 +73,23 @@ module PubSubModelSync
     end
 
     # convert payload data into Payload
-    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
       data = data.deep_symbolize_keys
-      new(data[:data], data[:attributes], data[:headers])
+      new(data[:data], data[:info], data[:headers])
     end
 
     private
 
     def build_headers
-      headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+      headers[:key] ||= [klass, action].join('/')
+      headers[:ordering_key] ||= klass
+      headers[:uuid] ||= SecureRandom.uuid
     end
 
     def validate!
-      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+      raise MissingInfo if !info[:klass] || !info[:action]
     end
   end
 end

data/lib/pub_sub_model_sync/publisher.rb

@@ -1,40 +1,57 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Publisher
-    attr_accessor :attrs, :actions, :klass, :as_klass
-
-    def initialize(attrs, klass, actions = nil, as_klass = nil)
-      @attrs = attrs
-      @klass = klass
-      @actions = actions || %i[create update destroy]
-      @as_klass = as_klass || klass
+  class Publisher < PubSubModelSync::Base
+    attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
+
+    # @param model (ActiveRecord::Base)
+    # @param action (@see PublishConcern::ps_publish)
+    # @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
+    def initialize(model, action, settings = {})
+      @model = model
+      @action = action
+      @data = settings[:data] || {}
+      @mapping = settings[:mapping] || []
+      @headers = settings[:headers] || {}
+      @as_klass = settings[:as_klass] || model.class.name
+    end
+
+    # @return (Payload)
+    def payload
+      values = compute_value(data)
+      values = mapping_data.merge(values)
+      PubSubModelSync::Payload.new(values, settings_data, headers_data)
     end
 
-    # Builds the payload with model information defined for :action (:create|:update|:destroy)
-    def payload(model, action)
-      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
+    def self.ordering_key_for(model)
+      [model.class.name, model.id || SecureRandom.uuid].join('/')
     end
 
     private
 
-    def payload_data(model)
-      source_props = @attrs.map { |prop| prop.to_s.split(':').first }
-      data = model.as_json(only: source_props, methods: source_props)
-      aliased_props = @attrs.select { |prop| prop.to_s.include?(':') }
-      aliased_props.each do |prop|
-        source, target = prop.to_s.split(':')
-        data[target] = data.delete(source)
-      end
-      data.symbolize_keys
+    def headers_data
+      klass_name = model.class.name
+      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      def_data.merge(compute_value(headers))
     end
 
-    def payload_attrs(model, action)
-      {
-        klass: (as_klass || model.class.name).to_s,
-        action: action.to_sym,
-        key: [model.class.name, action, model.id].join('/')
-      }
+    def compute_value(value)
+      res = value
+      res = model.send(value, action) if value.is_a?(Symbol) # method name
+      res = value.call(model, action) if value.is_a?(Proc)
+      res
+    end
+
+    def settings_data
+      { klass: as_klass, action: action }
+    end
+
+    def mapping_data
+      mapping.map do |prop|
+        source, target = prop.to_s.split(':')
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
     end
   end
 end