pub_sub_model_sync 0.5.10 → 1.0

data/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: '2'
+
+services:
+  test:
+    build: .
+    command: sh -c 'bundle exec rspec'
+    volumes:
+      - .:/app
+      - bundler_gems:/usr/local/bundle/
+
+volumes:
+  bundler_gems:

data/lib/pub_sub_model_sync.rb

@@ -10,10 +10,12 @@ require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'
 require 'pub_sub_model_sync/runner'
+require 'pub_sub_model_sync/transaction'
 require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
+require 'pub_sub_model_sync/run_subscriber'
 
-require 'pub_sub_model_sync/publisher'
+require 'pub_sub_model_sync/payload_builder'
 require 'pub_sub_model_sync/subscriber'
 
 require 'pub_sub_model_sync/service_base'

data/lib/pub_sub_model_sync/base.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Base
-    delegate :config, :log, to: self
+    delegate :config, :log, :debug?, to: self
 
     class << self
       def config
@@ -12,13 +12,10 @@ module PubSubModelSync
       def log(message, kind = :info)
         config.log message, kind
       end
-    end
 
-    def retry_error(error_klass, qty: 2, &block)
-      retries ||= 0
-      block.call
-    rescue error_klass => _e
-      (retries += 1) <= qty ? retry : raise
+      def debug?
+        config.debug
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -3,12 +3,13 @@
 module PubSubModelSync
   class Config
     cattr_accessor(:subscribers) { [] }
-    cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
 
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
+    cattr_accessor(:transactions_max_buffer) { 100 }
+    cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -16,29 +17,37 @@ module PubSubModelSync
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
     cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
-    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
-    cattr_accessor :project, :credentials, :topic_name, :subscription_name
+    cattr_accessor :project, :credentials, :topic_name, :subscription_name, :default_topic_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name, :default_topic_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name, :subscription_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name, :default_topic_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
       if logger == :raise_error
-        kind == :error ? raise(msg) : puts(msg)
+        kind == :error ? raise(StandardError, msg) : puts(msg)
       else
         logger ? logger.send(kind, msg) : puts(msg)
       end
     end
 
     def self.subscription_key
-      subscription_name ||
-        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+      klass = Rails.application.class
+      app_name = klass.respond_to?(:module_parent_name) ? klass.module_parent_name : klass.parent_name
+      subscription_name || app_name
+    end
+
+    class << self
+      alias default_topic_name_old default_topic_name
+
+      def default_topic_name
+        default_topic_name_old || Array(topic_name).first
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/initializers/before_commit.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
+ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
+  alias_method :commit_without_before_commit, :commit
+
+  def commit
+    call_before_commit_records if PubSubModelSync::Config.enable_rails4_before_commit
+    commit_without_before_commit
+  end
+
+  private
+
+  def call_before_commit_records
+    ite = records.uniq
+    ite.each do |record|
+      action = record.previous_changes.include?(:id) ? :create : :update
+      action = :destroy if record.destroyed?
+      callback_name = "ps_before_#{action}_commit".to_sym
+      record.send(callback_name) if record.respond_to?(callback_name)
+    end
+  end
+end

data/lib/pub_sub_model_sync/message_processor.rb

@@ -16,25 +16,29 @@ module PubSubModelSync
     end
 
     def process!
-      filter_subscribers.each(&method(:run_subscriber))
+      subscribers = filter_subscribers
+      payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
+      log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
+      subscribers.each(&method(:run_subscriber))
     end
 
     def process
+      retries ||= 0
       process!
     rescue => e
-      notify_error(e)
+      retry_process?(e, retries += 1) ? retry : notify_error(e)
     end
 
     private
 
     def run_subscriber(subscriber)
+      processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
-        subscriber.process!(payload)
-        res = config.on_success_processing.call(payload, { subscriber: subscriber })
-        log "processed message with: #{payload.inspect}" if res != :skip_log
-      end
+      log("Processing message #{[subscriber, payload]}...") if config.debug
+      processor.call
+      res = config.on_success_processing.call(payload, { subscriber: subscriber })
+      log "processed message with: #{payload.inspect}" if res != :skip_log
     end
 
     def processable?(subscriber)
@@ -43,17 +47,37 @@ module PubSubModelSync
       !cancel
     end
 
-    # @param error (Error)
+    # @param error (StandardError)
     def notify_error(error)
       info = [payload, error.message, error.backtrace]
       res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 
+    def lost_db_connection?(error)
+      connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
+      connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
+    end
+
+    def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
+      error_payload = [payload, error.message, error.backtrace]
+      return false unless lost_db_connection?(error)
+
+      if retries <= 5
+        sleep(retries)
+        log("Error processing message: (retrying #{retries}/5): #{error_payload}", :error)
+        ActiveRecord::Base.connection.reconnect! rescue nil # rubocop:disable Style/RescueModifier
+        true
+      else
+        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
+        Process.exit!(true)
+      end
+    end
+
+    # @return (Array<PubSubModelSync::Subscriber>)
     def filter_subscribers
       config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
-          subscriber.settings[:from_action].to_s == payload.action.to_s
+        subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
       end
     end
   end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -3,65 +3,126 @@
 module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
+      class MissingPublisher < StandardError; end
+      attr_accessor :current_transaction
+
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
-      # Publishes any value to pubsub
-      # @param klass (String): Class name
-      # @param data (Hash): Data to be delivered
-      # @param action (:symbol): action name
-      def publish_data(klass, data, action)
-        attrs = { klass: klass.to_s, action: action.to_sym }
-        payload = PubSubModelSync::Payload.new(data, attrs)
-        publish(payload)
+      # Permits to group all payloads with the same ordering_key and be processed in the same order
+      #   they are published by the subscribers. Grouping by ordering_key allows us to enable
+      #   multiple workers in our Pub/Sub service(s), and still guarantee that related payloads will
+      #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
+      #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
+      #   worker/thread.
+      # @see Transaction.new(...)
+      # @param key (String|Nil)
+      # @param block (Yield) block to be executed
+      def transaction(key, settings = {}, &block)
+        t = init_transaction(key, settings)
+        block.call
+        t.finish
+      rescue
+        t.rollback
+        raise
+      ensure
+        t.clean_publisher
       end
 
-      # Publishes model info to pubsub
-      # @param model (ActiveRecord model)
-      # @param action (Sym): Action name
-      # @param publisher (Publisher, optional): Publisher to be used
-      def publish_model(model, action, publisher = nil)
-        return if model.ps_skip_sync?(action)
-
-        publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload.data)
-        return if res_before == :cancel
+      # Starts a new transaction
+      # @param key (String|Nil)
+      # @return (Transaction)
+      def init_transaction(key, settings = {})
+        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        if current_transaction
+          current_transaction.add_transaction(new_transaction)
+        else
+          self.current_transaction = new_transaction
+        end
+        new_transaction
+      end
 
+      # Publishes a class level notification via pubsub
+      # @refer PublisherConcern.ps_class_publish
+      # @return Payload
+      def publish_data(klass, data, action, headers: {})
+        info = { klass: klass.to_s, action: action.to_sym, mode: :klass }
+        log("Building payload for: #{info.inspect}") if config.debug
+        payload = PubSubModelSync::Payload.new(data, info, headers)
+        define_transaction_key(payload)
         publish(payload)
-        model.ps_after_sync(action, payload.data)
+      end
+
+      # @param model (ActiveRecord::Base)
+      # @param action (Symbol: @see PublishConcern::ps_publish)
+      # @param settings (Hash: @see PayloadBuilder.settings)
+      def publish_model(model, action, settings = {})
+        log("Building payload for: #{[model, action].inspect}") if config.debug
+        payload = PubSubModelSync::PayloadBuilder.new(model, action, settings).call
+        define_transaction_key(payload)
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
+          publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
+        end
       end
 
       # Publishes payload to pubsub
-      # @attr payload (PubSubModelSync::Payload)
+      # @param payload (PubSubModelSync::Payload)
+      # @return Payload
       # Raises error if exist
-      def publish!(payload)
-        if config.on_before_publish.call(payload) == :cancel
-          log("Publish message cancelled: #{payload}") if config.debug
-          return
-        end
+      def publish!(payload, &block)
+        payload.headers[:ordering_key] = ordering_key_for(payload)
+        return unless ensure_publish(payload)
 
-        log("Publishing message: #{[payload]}")
+        current_transaction ? current_transaction.add_payload(payload) : connector_publish(payload)
+        block&.call
+        payload
+      end
+
+      def connector_publish(payload)
+        log("Publishing message #{payload.inspect}...") if config.debug
         connector.publish(payload)
+        log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)
       end
 
       # Similar to :publish! method
       # Notifies error via :on_error_publish instead of raising error
-      def publish(payload)
-        publish!(payload)
+      # @return Payload
+      def publish(payload, &block)
+        publish!(payload, &block)
       rescue => e
         notify_error(e, payload)
       end
 
       private
 
+      def ensure_publish(payload)
+        cancelled = config.on_before_publish.call(payload) == :cancel
+        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
+      def ordering_key_for(payload)
+        payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
+      end
+
+      def ensure_model_publish(model, action, payload)
+        res_before = model.ps_before_publish(action, payload)
+        cancelled = res_before == :cancel
+        log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
+        !cancelled
+      end
+
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
         res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
+
+      def define_transaction_key(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -26,6 +26,10 @@ module PubSubModelSync
     end
 
     class MockTopic
+      def name
+        'name'
+      end
+
       def subscription(*_args)
         @subscription ||= MockSubscription.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -28,16 +28,29 @@ module PubSubModelSync
       def subscribe(*_args)
         true
       end
+
+      def mark_message_as_processed(*_args)
+        true
+      end
     end
 
     def producer(*_args)
       MockProducer.new
     end
+    alias async_producer producer
 
     def consumer(*_args)
       MockConsumer.new
     end
 
+    def topics
+      []
+    end
+
+    def create_topic(_name)
+      true
+    end
+
     def close
       true
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -3,30 +3,48 @@
 module PubSubModelSync
   class Payload
     class MissingInfo < StandardError; end
-    attr_reader :data, :attributes, :headers
+    attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass*: string, action*: :sym }):
-    # @param headers (Hash: { key?: string, ...any_key?: anything }):
-    def initialize(data, attributes, headers = {})
-      @data = data
-      @attributes = attributes
-      @headers = headers
+    # @param info (Hash):
+    #   klass: (String, required) Notification class name
+    #   action: (Symbol, required) Notification action name
+    #   mode: (:model|:klass, default :model): :model for instance and :klass for class notifications
+    # @param headers (Hash):
+    #   key (String): identifier of the payload, default:
+    #        <klass/action>: when class message
+    #        <klass/action/model.id>: when model message
+    #   ordering_key (String): messages with the same key are processed in the same order they
+    #     were delivered, default:
+    #        <klass>: when class message
+    #        <klass/id>: when model message
+    #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
+    #     message (default first topic)
+    #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
+    #     even withing transactions. Default nil.
+    def initialize(data, info, headers = {})
+      @data = data.deep_symbolize_keys
+      @info = info.deep_symbolize_keys
+      @headers = headers.deep_symbolize_keys
       build_headers
       validate!
     end
 
     # @return Hash: payload data
     def to_h
-      { data: data, attributes: attributes, headers: headers }
+      { data: data.clone, info: info.clone, headers: headers.clone }
     end
 
     def klass
-      attributes[:klass]
+      info[:klass].to_s
     end
 
     def action
-      attributes[:action]
+      info[:action].to_sym
+    end
+
+    def mode
+      (info[:mode] || :model).to_sym
     end
 
     # Process payload data
@@ -58,22 +76,23 @@ module PubSubModelSync
     end
 
     # convert payload data into Payload
-    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
-      data = data.deep_symbolize_keys
-      new(data[:data], data[:attributes], data[:headers])
+      data = data.symbolize_keys
+      new(data[:data], data[:info] || data[:attributes], data[:headers])
     end
 
     private
 
     def build_headers
-      headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
-      headers[:key] ||= [klass.to_s, action].join('/')
+      headers[:key] ||= [klass, action].join('/')
+      headers[:ordering_key] ||= klass
+      headers[:uuid] ||= SecureRandom.uuid
     end
 
     def validate!
-      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+      raise MissingInfo if !info[:klass] || !info[:action]
     end
   end
 end