pub_sub_model_sync 1.0.beta → 1.0.1

data/lib/pub_sub_model_sync.rb

@@ -15,7 +15,7 @@ require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
 require 'pub_sub_model_sync/run_subscriber'
 
-require 'pub_sub_model_sync/publisher'
+require 'pub_sub_model_sync/payload_builder'
 require 'pub_sub_model_sync/subscriber'
 
 require 'pub_sub_model_sync/service_base'

data/lib/pub_sub_model_sync/base.rb

@@ -17,25 +17,5 @@ module PubSubModelSync
         config.debug
       end
     end
-
-    # @param errors (Array(Class|String))
-    def retry_error(errors, qty: 2, &block)
-      retries ||= 0
-      block.call
-    rescue => e
-      retries += 1
-      res = errors.find { |e_type| match_error?(e, e_type) }
-      raise if !res || retries > qty
-
-      sleep(qty * 0.1) && retry
-    end
-
-    private
-
-    # @param error (Exception)
-    # @param error_type (Class|String)
-    def match_error?(error, error_type)
-      error_type.is_a?(String) ? error.message.include?(error_type) : error.is_a?(error_type)
-    end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -8,7 +8,8 @@ module PubSubModelSync
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
-    cattr_accessor(:transactions_use_buffer) { true }
+    cattr_accessor(:transactions_max_buffer) { 100 }
+    cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -29,7 +30,7 @@ module PubSubModelSync
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
       if logger == :raise_error
-        kind == :error ? raise(msg) : puts(msg)
+        kind == :error ? raise(StandardError, msg) : puts(msg)
       else
         logger ? logger.send(kind, msg) : puts(msg)
       end

data/lib/pub_sub_model_sync/initializers/before_commit.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
+ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
+  alias_method :commit_without_before_commit, :commit
+
+  def commit
+    call_before_commit_records if PubSubModelSync::Config.enable_rails4_before_commit
+    commit_without_before_commit
+  end
+
+  private
+
+  def call_before_commit_records
+    ite = records.uniq
+    ite.each do |record|
+      action = record.previous_changes.include?(:id) ? :create : :update
+      action = :destroy if record.destroyed?
+      callback_name = "ps_before_#{action}_commit".to_sym
+      record.send(callback_name) if record.respond_to?(callback_name)
+    end
+  end
+end

data/lib/pub_sub_model_sync/message_processor.rb

@@ -16,13 +16,17 @@ module PubSubModelSync
     end
 
     def process!
-      filter_subscribers.each(&method(:run_subscriber))
+      subscribers = filter_subscribers
+      payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
+      log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
+      subscribers.each(&method(:run_subscriber))
     end
 
     def process
+      retries ||= 0
       process!
     rescue => e
-      notify_error(e)
+      retry_process?(e, retries += 1) ? retry : notify_error(e)
     end
 
     private
@@ -31,12 +35,10 @@ module PubSubModelSync
       processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
-      retry_error(errors, qty: 5) do
-        processor.call
-        res = config.on_success_processing.call(payload, { subscriber: subscriber })
-        log "processed message with: #{payload.inspect}" if res != :skip_log
-      end
+      log("Processing message #{[subscriber, payload]}...") if config.debug
+      processor.call
+      res = config.on_success_processing.call(payload, { subscriber: subscriber })
+      log "processed message with: #{payload.inspect}" if res != :skip_log
     end
 
     def processable?(subscriber)
@@ -45,13 +47,34 @@ module PubSubModelSync
       !cancel
     end
 
-    # @param error (Error)
+    # @param error (StandardError)
     def notify_error(error)
       info = [payload, error.message, error.backtrace]
       res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 
+    def lost_db_connection?(error)
+      connection_lost_classes = %w[ActiveRecord::ConnectionTimeoutError PG::UnableToSend]
+      connection_lost_classes.include?(error.class.name) || error.message.match?(/lost connection/i)
+    end
+
+    def retry_process?(error, retries) # rubocop:disable Metrics/MethodLength
+      error_payload = [payload, error.message, error.backtrace]
+      return false unless lost_db_connection?(error)
+
+      if retries <= 5
+        sleep(retries)
+        log("Error processing message: (retrying #{retries}/5): #{error_payload}", :error)
+        ActiveRecord::Base.connection.reconnect! rescue nil # rubocop:disable Style/RescueModifier
+        true
+      else
+        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
+        Process.exit!(true)
+      end
+    end
+
+    # @return (Array<PubSubModelSync::Subscriber>)
     def filter_subscribers
       config.subscribers.select do |subscriber|
         subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -22,7 +22,7 @@ module PubSubModelSync
       def transaction(key, settings = {}, &block)
         t = init_transaction(key, settings)
         block.call
-        t.deliver_all
+        t.finish
       rescue
         t.rollback
         raise
@@ -31,7 +31,7 @@ module PubSubModelSync
       end
 
       # Starts a new transaction
-      # @param key (@transaction_key)
+      # @param key (String|Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
         new_transaction = PubSubModelSync::Transaction.new(key, settings)
@@ -47,20 +47,20 @@ module PubSubModelSync
       # @refer PublisherConcern.ps_class_publish
       # @return Payload
       def publish_data(klass, data, action, headers: {})
-        attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
-        payload = PubSubModelSync::Payload.new(data, attrs, headers)
+        info = { klass: klass.to_s, action: action.to_sym, mode: :klass }
+        log("Building payload for: #{info.inspect}") if config.debug
+        payload = PubSubModelSync::Payload.new(data, info, headers)
+        define_transaction_key(payload)
         publish(payload)
       end
 
-      # @param model (ActiveRecord::Base)
-      # @param action (Symbol: @see PublishConcern::ps_publish)
-      # @param settings (Hash: @see Publisher.new.settings)
+      # @param model (ActiveRecord::Base,PubSubModelSync::PublisherConcern)
+      # @param action (Symbol,String @see PublishConcern::ps_publish)
+      # @param settings (Hash @see PayloadBuilder.settings)
       def publish_model(model, action, settings = {})
-        return if model.ps_skip_publish?(action)
-
-        publisher = PubSubModelSync::Publisher.new(model, action, settings)
-        payload = publisher.payload
-
+        log("Building payload for: #{[model, action].inspect}") if config.debug
+        payload = PubSubModelSync::PayloadBuilder.new(model, action, settings).call
+        define_transaction_key(payload)
         transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
           publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
         end
@@ -80,6 +80,7 @@ module PubSubModelSync
       end
 
       def connector_publish(payload)
+        log("Publishing message #{[payload]}...") if config.debug
         connector.publish(payload)
         log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)
@@ -98,19 +99,18 @@ module PubSubModelSync
 
       def ensure_publish(payload)
         cancelled = config.on_before_publish.call(payload) == :cancel
-        log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
+        log("Publish cancelled by config.on_before_publish: #{[payload]}") if config.debug && cancelled
         !cancelled
       end
 
       def ordering_key_for(payload)
-        current_transaction&.key ||= payload.headers[:ordering_key]
         payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
       end
 
       def ensure_model_publish(model, action, payload)
         res_before = model.ps_before_publish(action, payload)
         cancelled = res_before == :cancel
-        log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
+        log("Publish cancelled by model.ps_before_publish: #{[payload]}") if config.debug && cancelled
         !cancelled
       end
 
@@ -119,6 +119,10 @@ module PubSubModelSync
         res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
+
+      def define_transaction_key(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/payload.rb

@@ -6,30 +6,33 @@ module PubSubModelSync
     attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
+    # @param info (Hash):
+    #   klass: (String, required) Notification class name
+    #   action: (Symbol, required) Notification action name
+    #   mode: (:model|:klass, default :model): :model for instance and :klass for class notifications
     # @param headers (Hash):
     #   key (String): identifier of the payload, default:
-    #        klass/action: when class message
-    #        klass/action/model.id: when model message
+    #        <klass/action>: when class message
+    #        <klass/action/model.id>: when model message
     #   ordering_key (String): messages with the same key are processed in the same order they
     #     were delivered, default:
-    #        klass: when class message
-    #        klass/id: when model message
+    #        <klass>: when class message
+    #        <klass/id>: when model message
     #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
     #     message (default first topic)
     #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
     #     even withing transactions. Default nil.
     def initialize(data, info, headers = {})
-      @data = data
-      @info = { mode: :model }.merge(info)
-      @headers = headers
+      @data = data.deep_symbolize_keys
+      @info = info.deep_symbolize_keys
+      @headers = headers.deep_symbolize_keys
       build_headers
       validate!
     end
 
     # @return Hash: payload data
     def to_h
-      { data: data, info: info, headers: headers }
+      { data: data.clone, info: info.clone, headers: headers.clone }
     end
 
     def klass
@@ -41,7 +44,7 @@ module PubSubModelSync
     end
 
     def mode
-      info[:mode].to_sym
+      (info[:mode] || :model).to_sym
     end
 
     # Process payload data
@@ -75,8 +78,8 @@ module PubSubModelSync
     # convert payload data into Payload
     # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
-      data = data.deep_symbolize_keys
-      new(data[:data], data[:info], data[:headers])
+      data = data.symbolize_keys
+      new(data[:data], data[:info] || data[:attributes], data[:headers])
     end
 
     private

data/lib/pub_sub_model_sync/{publisher.rb → payload_builder.rb}

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Publisher < PubSubModelSync::Base
+  class PayloadBuilder < PubSubModelSync::Base
     attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
 
-    # @param model (ActiveRecord::Base)
+    # @param model (ActiveRecord::Base,PubSubModelSync::PublisherConcern)
     # @param action (@see PublishConcern::ps_publish)
     # @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
     def initialize(model, action, settings = {})
@@ -17,9 +17,9 @@ module PubSubModelSync
     end
 
     # @return (Payload)
-    def payload
+    def call
       values = compute_value(data)
-      values = mapping_data.merge(values)
+      values = self.class.parse_mapping_for(model, mapping).merge(values)
       PubSubModelSync::Payload.new(values, settings_data, headers_data)
     end
 
@@ -27,6 +27,18 @@ module PubSubModelSync
       [model.class.name, model.id || SecureRandom.uuid].join('/')
     end
 
+    # @param model (ActiveRecord::Base)
+    # @param mapping (@see PublishConcern::ps_publish -> mapping)
+    # @return (Hash) Hash with the corresponding values for each attribute
+    # Sample: parse_mapping_for(my_model, %w[id name:full_name])
+    #         ==> { id: 10, full_name: 'model.name value' }
+    def self.parse_mapping_for(model, mapping)
+      mapping.map do |prop|
+        source, target = prop.to_s.split(':')
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
+    end
+
     private
 
     def headers_data
@@ -46,12 +58,5 @@ module PubSubModelSync
     def settings_data
       { klass: as_klass, action: action }
     end
-
-    def mapping_data
-      mapping.map do |prop|
-        source, target = prop.to_s.split(':')
-        [target || source, model.send(source.to_sym)]
-      end.to_h.symbolize_keys
-    end
   end
 end

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -9,12 +9,6 @@ module PubSubModelSync
       ps_init_transaction_callbacks if self <= ActiveRecord::Base
     end
 
-    # before preparing data to sync
-    def ps_skip_publish?(_action)
-      false
-    end
-    alias ps_skip_sync? ps_skip_publish? # @deprecated
-
     # before delivering data (return :cancel to cancel sync)
     def ps_before_publish(_action, _payload); end
     alias ps_before_sync ps_before_publish # @deprecated
@@ -24,14 +18,14 @@ module PubSubModelSync
     alias ps_after_sync ps_after_publish # @deprecated
 
     # Delivers a notification via pubsub
-    # @param action (Sym|String) Sample: create|update|save|destroy|<any_other_key>
+    # @param action (Symbol,String) Sample: create|update|save|destroy|<any_other_key>
     # @param mapping? (Array<String>) If present will generate data using the mapping and added to the payload.
     #   Sample: ["id", "full_name:name"]
-    # @param data? (Hash|Symbol|Proc)
+    # @param data? (Hash,Symbol,Proc)
     #   Hash: Data to be added to the payload
     #   Symbol: Method name to be called to retrieve payload data (must return a hash value, receives :action name)
     #   Proc: Block to be called to retrieve payload data
-    # @param headers? (Hash|Symbol|Proc): (All available attributes in Payload.headers)
+    # @param headers? (Hash,Symbol,Proc): (All available attributes in @Payload.headers)
     #   Hash: Data that will be merged with default header values
     #   Symbol: Method name that will be called to retrieve header values (must return a hash, receives :action name)
     #   Proc: Block to be called to retrieve header values
@@ -42,6 +36,16 @@ module PubSubModelSync
     end
     delegate :ps_class_publish, to: :class
 
+    # Permits to perform manually the callback for a specific action
+    # @param action (Symbol, default: :create) Only :create|:update|:destroy
+    def ps_perform_publish(action = :create)
+      items = self.class.ps_cache_publish_callbacks.select { |item| item[:actions].include?(action) }
+      raise(StandardError, "No callback found for action :#{action}") if items.empty?
+
+      items.each { |item| instance_exec(action, &item[:callback]) }
+      self
+    end
+
     module ClassMethods
       # Publishes a class level notification via pubsub
       # @param data (Hash): Data of the notification
@@ -53,32 +57,48 @@ module PubSubModelSync
         klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
       end
 
-      # @param crud_actions (Symbol|Array<Symbol>): :create, :update, :destroy
+      # @param crud_actions (Symbol,Array<Symbol>): :create, :update, :destroy
       # @param method_name (Symbol, optional) method to be called
-      def ps_on_crud_event(crud_actions, method_name = nil, &block)
-        crud_actions = Array(crud_actions)
+      def ps_after_action(crud_actions, method_name = nil, &block)
+        actions = Array(crud_actions).map(&:to_sym)
         callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
-        commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
-        crud_actions.each do |action|
-          send(commit_name, on: :create) { instance_exec(action, &callback) } if action == :create
-          send(commit_name, on: :update) { instance_exec(action, &callback) } if action == :update
-          after_destroy { instance_exec(action, &callback) } if action == :destroy
+        ps_cache_publish_callbacks({ actions: actions, callback: callback })
+        actions.each do |action|
+          if action == :destroy
+            after_destroy { instance_exec(action, &callback) }
+          else
+            ps_define_commit_action(action, callback)
+          end
         end
       end
 
+      def ps_cache_publish_callbacks(new_value = nil)
+        @ps_cache_publish_callbacks ||= []
+        @ps_cache_publish_callbacks << new_value if new_value
+        @ps_cache_publish_callbacks
+      end
+
       private
 
+      def ps_define_commit_action(action, callback)
+        if PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
+          define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
+        else
+          commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
+          send(commit_name, on: action) { instance_exec(action, &callback) }
+        end
+      end
+
       # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
       def ps_init_transaction_callbacks
         start_transaction = lambda do
-          key = id ? PubSubModelSync::Publisher.ordering_key_for(self) : nil
-          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(key)
+          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(nil)
         end
-        after_create start_transaction, prepend: true # wait for ID
+        before_create start_transaction, prepend: true
         before_update start_transaction, prepend: true
         before_destroy start_transaction, prepend: true
-        after_commit { @ps_transaction.deliver_all }
-        after_rollback(prepend: true) { @ps_transaction.rollback }
+        after_commit { @ps_transaction&.finish }
+        after_rollback(prepend: true) { @ps_transaction&.rollback }
       end
     end
   end