pub_sub_model_sync 1.0.beta2 → 1.0

data/lib/pub_sub_model_sync.rb

@@ -15,7 +15,7 @@ require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
 require 'pub_sub_model_sync/run_subscriber'
 
-require 'pub_sub_model_sync/publisher'
+require 'pub_sub_model_sync/payload_builder'
 require 'pub_sub_model_sync/subscriber'
 
 require 'pub_sub_model_sync/service_base'

data/lib/pub_sub_model_sync/base.rb

@@ -17,25 +17,5 @@ module PubSubModelSync
         config.debug
       end
     end
-
-    # @param errors (Array(Class|String))
-    def retry_error(errors, qty: 2, &block)
-      retries ||= 0
-      block.call
-    rescue => e
-      retries += 1
-      res = errors.find { |e_type| match_error?(e, e_type) }
-      raise if !res || retries > qty
-
-      sleep(qty * 0.1) && retry
-    end
-
-    private
-
-    # @param error (Exception)
-    # @param error_type (Class|String)
-    def match_error?(error, error_type)
-      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
-    end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -30,7 +30,7 @@ module PubSubModelSync
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
       if logger == :raise_error
-        kind == :error ? raise(msg) : puts(msg)
+        kind == :error ? raise(StandardError, msg) : puts(msg)
       else
         logger ? logger.send(kind, msg) : puts(msg)
       end

data/lib/pub_sub_model_sync/initializers/before_commit.rb

@@ -2,11 +2,11 @@
 
 # Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
 ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
-  alias_method :commit_with_before_commit, :commit
+  alias_method :commit_without_before_commit, :commit
 
   def commit
-    call_before_commit_records if Rails::VERSION::MAJOR == 4
-    commit_with_before_commit
+    call_before_commit_records if PubSubModelSync::Config.enable_rails4_before_commit
+    commit_without_before_commit
   end
 
   private

data/lib/pub_sub_model_sync/message_processor.rb

@@ -16,13 +16,17 @@ module PubSubModelSync
     end
 
     def process!
-      filter_subscribers.each(&method(:run_subscriber))
+      subscribers = filter_subscribers
+      payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
+      log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
+      subscribers.each(&method(:run_subscriber))
     end
 
     def process
+      retries ||= 0
       process!
     rescue => e
-      notify_error(e)
+      retry_process?(e, retries += 1) ? retry : notify_error(e)
     end
 
     private
@@ -31,12 +35,10 @@ module PubSubModelSync
       processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
-      retry_error(errors, qty: 5) do
-        processor.call
-        res = config.on_success_processing.call(payload, { subscriber: subscriber })
-        log "processed message with: #{payload.inspect}" if res != :skip_log
-      end
+      log("Processing message #{[subscriber, payload]}...") if config.debug
+      processor.call
+      res = config.on_success_processing.call(payload, { subscriber: subscriber })
+      log "processed message with: #{payload.inspect}" if res != :skip_log
     end
 
     def processable?(subscriber)
@@ -45,13 +47,34 @@ module PubSubModelSync
       !cancel
     end
 
-    # @param error (Error)
+    # @param error (StandardError)
     def notify_error(error)
       info = [payload, error.message, error.backtrace]
       res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 
+    def lost_db_connection?(error)
+      connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
+      connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
+    end
+
+    def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
+      error_payload = [payload, error.message, error.backtrace]
+      return false unless lost_db_connection?(error)
+
+      if retries <= 5
+        sleep(retries)
+        log("Error processing message: (retrying #{retries}/5): #{error_payload}", :error)
+        ActiveRecord::Base.connection.reconnect! rescue nil # rubocop:disable Style/RescueModifier
+        true
+      else
+        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
+        Process.exit!(true)
+      end
+    end
+
+    # @return (Array<PubSubModelSync::Subscriber>)
     def filter_subscribers
       config.subscribers.select do |subscriber|
         subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -31,7 +31,7 @@ module PubSubModelSync
       end
 
       # Starts a new transaction
-      # @param key (@transaction_key)
+      # @param key (String|Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
         new_transaction = PubSubModelSync::Transaction.new(key, settings)
@@ -47,20 +47,20 @@ module PubSubModelSync
       # @refer PublisherConcern.ps_class_publish
       # @return Payload
       def publish_data(klass, data, action, headers: {})
-        attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
-        payload = PubSubModelSync::Payload.new(data, attrs, headers)
+        info = { klass: klass.to_s, action: action.to_sym, mode: :klass }
+        log("Building payload for: #{info.inspect}") if config.debug
+        payload = PubSubModelSync::Payload.new(data, info, headers)
+        define_transaction_key(payload)
         publish(payload)
       end
 
       # @param model (ActiveRecord::Base)
       # @param action (Symbol: @see PublishConcern::ps_publish)
-      # @param settings (Hash: @see Publisher.new.settings)
+      # @param settings (Hash: @see PayloadBuilder.settings)
       def publish_model(model, action, settings = {})
-        return if model.ps_skip_publish?(action)
-
-        publisher = PubSubModelSync::Publisher.new(model, action, settings)
-        payload = publisher.payload
-
+        log("Building payload for: #{[model, action].inspect}") if config.debug
+        payload = PubSubModelSync::PayloadBuilder.new(model, action, settings).call
+        define_transaction_key(payload)
         transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
           publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
         end
@@ -80,6 +80,7 @@ module PubSubModelSync
       end
 
       def connector_publish(payload)
+        log("Publishing message #{payload.inspect}...") if config.debug
         connector.publish(payload)
         log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)
@@ -103,7 +104,6 @@ module PubSubModelSync
       end
 
       def ordering_key_for(payload)
-        current_transaction&.key ||= payload.headers[:ordering_key]
         payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
       end
 
@@ -119,6 +119,10 @@ module PubSubModelSync
         res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
+
+      def define_transaction_key(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/payload.rb

@@ -6,30 +6,33 @@ module PubSubModelSync
     attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
+    # @param info (Hash):
+    #   klass: (String, required) Notification class name
+    #   action: (Symbol, required) Notification action name
+    #   mode: (:model|:klass, default :model): :model for instance and :klass for class notifications
     # @param headers (Hash):
     #   key (String): identifier of the payload, default:
-    #        klass/action: when class message
-    #        klass/action/model.id: when model message
+    #        <klass/action>: when class message
+    #        <klass/action/model.id>: when model message
     #   ordering_key (String): messages with the same key are processed in the same order they
     #     were delivered, default:
-    #        klass: when class message
-    #        klass/id: when model message
+    #        <klass>: when class message
+    #        <klass/id>: when model message
     #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
     #     message (default first topic)
     #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
     #     even withing transactions. Default nil.
     def initialize(data, info, headers = {})
-      @data = data
-      @info = { mode: :model }.merge(info)
-      @headers = headers
+      @data = data.deep_symbolize_keys
+      @info = info.deep_symbolize_keys
+      @headers = headers.deep_symbolize_keys
       build_headers
       validate!
     end
 
     # @return Hash: payload data
     def to_h
-      { data: data, info: info, headers: headers }
+      { data: data.clone, info: info.clone, headers: headers.clone }
     end
 
     def klass
@@ -41,7 +44,7 @@ module PubSubModelSync
     end
 
     def mode
-      info[:mode].to_sym
+      (info[:mode] || :model).to_sym
     end
 
     # Process payload data
@@ -75,8 +78,8 @@ module PubSubModelSync
     # convert payload data into Payload
     # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
-      data = data.deep_symbolize_keys
-      new(data[:data], data[:info], data[:headers])
+      data = data.symbolize_keys
+      new(data[:data], data[:info] || data[:attributes], data[:headers])
     end
 
     private

data/lib/pub_sub_model_sync/{publisher.rb → payload_builder.rb}

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Publisher < PubSubModelSync::Base
+  class PayloadBuilder < PubSubModelSync::Base
     attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
 
     # @param model (ActiveRecord::Base)
@@ -17,9 +17,9 @@ module PubSubModelSync
     end
 
     # @return (Payload)
-    def payload
+    def call
       values = compute_value(data)
-      values = mapping_data.merge(values)
+      values = self.class.parse_mapping_for(model, mapping).merge(values)
       PubSubModelSync::Payload.new(values, settings_data, headers_data)
     end
 
@@ -27,6 +27,18 @@ module PubSubModelSync
       [model.class.name, model.id || SecureRandom.uuid].join('/')
     end
 
+    # @param model (ActiveRecord::Base)
+    # @param mapping (@see PublishConcern::ps_publish -> mapping)
+    # @return (Hash) Hash with the corresponding values for each attribute
+    # Sample: parse_mapping_for(my_model, %w[id name:full_name])
+    #         ==> { id: 10, full_name: 'model.name value' }
+    def self.parse_mapping_for(model, mapping)
+      mapping.map do |prop|
+        source, target = prop.to_s.split(':')
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
+    end
+
     private
 
     def headers_data
@@ -46,12 +58,5 @@ module PubSubModelSync
     def settings_data
       { klass: as_klass, action: action }
     end
-
-    def mapping_data
-      mapping.map do |prop|
-        source, target = prop.to_s.split(':')
-        [target || source, model.send(source.to_sym)]
-      end.to_h.symbolize_keys
-    end
   end
 end

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -9,12 +9,6 @@ module PubSubModelSync
       ps_init_transaction_callbacks if self <= ActiveRecord::Base
     end
 
-    # before preparing data to sync
-    def ps_skip_publish?(_action)
-      false
-    end
-    alias ps_skip_sync? ps_skip_publish? # @deprecated
-
     # before delivering data (return :cancel to cancel sync)
     def ps_before_publish(_action, _payload); end
     alias ps_before_sync ps_before_publish # @deprecated
@@ -42,6 +36,16 @@ module PubSubModelSync
     end
     delegate :ps_class_publish, to: :class
 
+    # Permits to perform manually the callback for a specific action
+    # @param action (Symbol, default: :create) Only :create|:update|:destroy
+    def ps_perform_publish(action = :create)
+      items = self.class.ps_cache_publish_callbacks.select { |item| item[:actions].include?(action) }
+      raise(StandardError, "No callback found for action :#{action}") if items.empty?
+
+      items.each { |item| instance_exec(action, &item[:callback]) }
+      self
+    end
+
     module ClassMethods
       # Publishes a class level notification via pubsub
       # @param data (Hash): Data of the notification
@@ -55,24 +59,32 @@ module PubSubModelSync
 
       # @param crud_actions (Symbol|Array<Symbol>): :create, :update, :destroy
       # @param method_name (Symbol, optional) method to be called
-      def ps_on_crud_event(crud_actions, method_name = nil, &block)
+      def ps_after_action(crud_actions, method_name = nil, &block)
+        actions = Array(crud_actions).map(&:to_sym)
         callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
-        Array(crud_actions).each do |action|
+        ps_cache_publish_callbacks({ actions: actions, callback: callback })
+        actions.each do |action|
           if action == :destroy
             after_destroy { instance_exec(action, &callback) }
           else
-            ps_crud_define_commit_action(action, callback)
+            ps_define_commit_action(action, callback)
           end
         end
       end
 
+      def ps_cache_publish_callbacks(new_value = nil)
+        @ps_cache_publish_callbacks ||= []
+        @ps_cache_publish_callbacks << new_value if new_value
+        @ps_cache_publish_callbacks
+      end
+
       private
 
-      def ps_crud_define_commit_action(action, callback)
-        commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
+      def ps_define_commit_action(action, callback)
         if PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
           define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
         else
+          commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
           send(commit_name, on: action) { instance_exec(action, &callback) }
         end
       end
@@ -80,14 +92,13 @@ module PubSubModelSync
       # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
       def ps_init_transaction_callbacks
         start_transaction = lambda do
-          key = id ? PubSubModelSync::Publisher.ordering_key_for(self) : nil
-          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(key)
+          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(nil)
         end
-        after_create start_transaction, prepend: true # wait for ID
+        before_create start_transaction, prepend: true
         before_update start_transaction, prepend: true
         before_destroy start_transaction, prepend: true
-        after_commit { @ps_transaction.finish }
-        after_rollback(prepend: true) { @ps_transaction.rollback }
+        after_commit { @ps_transaction&.finish }
+        after_rollback(prepend: true) { @ps_transaction&.rollback }
       end
     end
   end

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -26,7 +26,7 @@ module PubSubModelSync
     def run_class_message
       model_class = subscriber.klass.constantize
       model_class.ps_processing_payload = payload # TODO: review for parallel notifications
-      call_action(model_class, payload.data) if ensure_sync(model_class)
+      call_action(model_class) if ensure_sync(model_class)
     end
 
     # support for: create, update, destroy
@@ -48,16 +48,17 @@ module PubSubModelSync
       res
     end
 
-    def call_action(object, *args)
-      action_name = settings[:to_action]
-      if action_name.is_a?(Proc)
-        args.prepend(object) unless klass_subscription?
-        action_name.call(*args)
-      else # method name
-        action_name = :save if %i[create update].include?(action_name.to_sym)
-        object.send(action_name, *args)
-      end
-      raise(object.errors) if object.respond_to?(:errors) && object.errors.any?
+    def call_action(object)
+      callback = settings[:to_action]
+      callback.is_a?(Proc) ? object.instance_exec(payload.data, &callback) : call_action_method(object)
+    end
+
+    def call_action_method(object)
+      method_name = settings[:to_action]
+      method_name = :save! if %i[create update].include?(method_name.to_sym)
+      method_name = :destroy! if method_name.to_sym == :destroy
+      is_crud_action = %i[save! destroy!].include?(method_name)
+      is_crud_action ? object.send(method_name) : object.send(method_name, payload.data)
     end
 
     def parse_condition(condition, object)
@@ -74,10 +75,13 @@ module PubSubModelSync
       model_class = subscriber.klass.constantize
       return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
 
+      error_msg = 'No values provided for identifiers:'
+      raise(StandardError, "#{error_msg} #{[settings[:id], payload]}") if model_identifiers.empty?
+
       model_class.where(model_identifiers).first_or_initialize
     end
 
-    # @param mappings (Array<String>) supports aliasing, sample: ["id", "full_name:name"]
+    # @param mappings (Array<String,Symbol>) supports aliasing, sample: ["id", "full_name:name"]
     # @return (Hash) hash with the correct attr names and its values
     def parse_mapping(mappings)
       mappings.map do |prop|
@@ -91,7 +95,7 @@ module PubSubModelSync
 
     # @return (Hash) hash including identifiers and its values
     def model_identifiers
-      @model_identifiers ||= parse_mapping(Array(settings[:id]).map(&:to_s))
+      @model_identifiers ||= parse_mapping(Array(settings[:id]))
     end
 
     def populate_model