pub_sub_model_sync 0.5.10 → 1.0

data/lib/pub_sub_model_sync/payload_builder.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class PayloadBuilder < PubSubModelSync::Base
+    attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
+
+    # @param model (ActiveRecord::Base)
+    # @param action (@see PublishConcern::ps_publish)
+    # @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
+    def initialize(model, action, settings = {})
+      @model = model
+      @action = action
+      @data = settings[:data] || {}
+      @mapping = settings[:mapping] || []
+      @headers = settings[:headers] || {}
+      @as_klass = settings[:as_klass] || model.class.name
+    end
+
+    # @return (Payload)
+    def call
+      values = compute_value(data)
+      values = self.class.parse_mapping_for(model, mapping).merge(values)
+      PubSubModelSync::Payload.new(values, settings_data, headers_data)
+    end
+
+    def self.ordering_key_for(model)
+      [model.class.name, model.id || SecureRandom.uuid].join('/')
+    end
+
+    # @param model (ActiveRecord::Base)
+    # @param mapping (@see PublishConcern::ps_publish -> mapping)
+    # @return (Hash) Hash with the corresponding values for each attribute
+    # Sample: parse_mapping_for(my_model, %w[id name:full_name])
+    #         ==> { id: 10, full_name: 'model.name value' }
+    def self.parse_mapping_for(model, mapping)
+      mapping.map do |prop|
+        source, target = prop.to_s.split(':')
+        [target || source, model.send(source.to_sym)]
+      end.to_h.symbolize_keys
+    end
+
+    private
+
+    def headers_data
+      klass_name = model.class.name
+      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      def_data.merge(compute_value(headers))
+    end
+
+    def compute_value(value)
+      res = value
+      res = model.send(value, action) if value.is_a?(Symbol) # method name
+      res = value.call(model, action) if value.is_a?(Proc)
+      res
+    end
+
+    def settings_data
+      { klass: as_klass, action: action }
+    end
+  end
+end

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -2,73 +2,103 @@
 
 module PubSubModelSync
   module PublisherConcern
-    def self.included(base)
-      base.extend(ClassMethods)
-    end
-
-    # Before initializing sync service (callbacks: after create/update/destroy)
-    def ps_skip_callback?(_action)
-      false
-    end
+    extend ActiveSupport::Concern
 
-    # before preparing data to sync
-    def ps_skip_sync?(_action)
-      false
+    included do
+      extend ClassMethods
+      ps_init_transaction_callbacks if self <= ActiveRecord::Base
     end
 
     # before delivering data (return :cancel to cancel sync)
-    def ps_before_sync(_action, _data); end
+    def ps_before_publish(_action, _payload); end
+    alias ps_before_sync ps_before_publish # @deprecated
 
     # after delivering data
-    def ps_after_sync(_action, _data); end
+    def ps_after_publish(_action, _payload); end
+    alias ps_after_sync ps_after_publish # @deprecated
+
+    # Delivers a notification via pubsub
+    # @param action (Sym|String) Sample: create|update|save|destroy|<any_other_key>
+    # @param mapping? (Array<String>) If present will generate data using the mapping and added to the payload.
+    #   Sample: ["id", "full_name:name"]
+    # @param data? (Hash|Symbol|Proc)
+    #   Hash: Data to be added to the payload
+    #   Symbol: Method name to be called to retrieve payload data (must return a hash value, receives :action name)
+    #   Proc: Block to be called to retrieve payload data
+    # @param headers? (Hash|Symbol|Proc): (All available attributes in Payload.headers)
+    #   Hash: Data that will be merged with default header values
+    #   Symbol: Method name that will be called to retrieve header values (must return a hash, receives :action name)
+    #   Proc: Block to be called to retrieve header values
+    # @param as_klass? (String): Output class name used instead of current class name
+    def ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: self.class.name)
+      p_klass = PubSubModelSync::MessagePublisher
+      p_klass.publish_model(self, action, data: data, mapping: mapping, headers: headers, as_klass: as_klass)
+    end
+    delegate :ps_class_publish, to: :class
+
+    # Permits to perform manually the callback for a specific action
+    # @param action (Symbol, default: :create) Only :create|:update|:destroy
+    def ps_perform_publish(action = :create)
+      items = self.class.ps_cache_publish_callbacks.select { |item| item[:actions].include?(action) }
+      raise(StandardError, "No callback found for action :#{action}") if items.empty?
 
-    # To perform sync on demand
-    # @param attrs (Array, optional): custom attrs to be used
-    # @param as_klass (Array, optional): custom klass name to be used
-    # @param publisher (Publisher, optional): custom publisher object
-    def ps_perform_sync(action = :create, attrs: nil, as_klass: nil,
-                        publisher: nil)
-      publisher ||= self.class.ps_publisher(action).dup
-      publisher.attrs = attrs if attrs
-      publisher.as_klass = as_klass if as_klass
-      PubSubModelSync::MessagePublisher.publish_model(self, action, publisher)
+      items.each { |item| instance_exec(action, &item[:callback]) }
+      self
     end
 
     module ClassMethods
-      # Permit to configure to publish crud actions (:create, :update, :destroy)
-      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil)
-        klass = PubSubModelSync::Publisher
-        publisher = klass.new(attrs, name, actions, as_klass)
-        PubSubModelSync::Config.publishers << publisher
+      # Publishes a class level notification via pubsub
+      # @param data (Hash): Data of the notification
+      # @param action (Symbol): action  name of the notification
+      # @param as_klass (String, default current class name): Class name of the notification
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      def ps_class_publish(data, action:, as_klass: nil, headers: {})
+        klass = PubSubModelSync::MessagePublisher
+        klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
+      end
+
+      # @param crud_actions (Symbol|Array<Symbol>): :create, :update, :destroy
+      # @param method_name (Symbol, optional) method to be called
+      def ps_after_action(crud_actions, method_name = nil, &block)
+        actions = Array(crud_actions).map(&:to_sym)
+        callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
+        ps_cache_publish_callbacks({ actions: actions, callback: callback })
         actions.each do |action|
-          ps_register_callback(action.to_sym, publisher)
+          if action == :destroy
+            after_destroy { instance_exec(action, &callback) }
+          else
+            ps_define_commit_action(action, callback)
+          end
         end
       end
 
-      # On demand class level publisher
-      def ps_class_publish(data, action:, as_klass: nil)
-        as_klass = (as_klass || name).to_s
-        klass = PubSubModelSync::MessagePublisher
-        klass.publish_data(as_klass, data, action.to_sym)
+      def ps_cache_publish_callbacks(new_value = nil)
+        @ps_cache_publish_callbacks ||= []
+        @ps_cache_publish_callbacks << new_value if new_value
+        @ps_cache_publish_callbacks
       end
 
-      # Publisher info for specific action
-      def ps_publisher(action = :create)
-        PubSubModelSync::Config.publishers.find do |publisher|
-          publisher.klass == name && publisher.actions.include?(action)
+      private
+
+      def ps_define_commit_action(action, callback)
+        if PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
+          define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
+        else
+          commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
+          send(commit_name, on: action) { instance_exec(action, &callback) }
         end
       end
 
-      private
-
-      def ps_register_callback(action, publisher)
-        after_commit(on: action) do |model|
-          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
-          if !disabled && !model.ps_skip_callback?(action)
-            klass = PubSubModelSync::MessagePublisher
-            klass.publish_model(model, action.to_sym, publisher)
-          end
+      # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
+      def ps_init_transaction_callbacks
+        start_transaction = lambda do
+          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(nil)
         end
+        before_create start_transaction, prepend: true
+        before_update start_transaction, prepend: true
+        before_destroy start_transaction, prepend: true
+        after_commit { @ps_transaction&.finish }
+        after_rollback(prepend: true) { @ps_transaction&.rollback }
       end
     end
   end

data/lib/pub_sub_model_sync/railtie.rb

@@ -2,6 +2,8 @@
 
 require 'pub_sub_model_sync'
 require 'rails'
+require 'active_record'
+require 'pub_sub_model_sync/config'
 module PubSubModelSync
   class Railtie < ::Rails::Railtie
     railtie_name :pub_sub_model_sync
@@ -9,5 +11,9 @@ module PubSubModelSync
     rake_tasks do
       load 'pub_sub_model_sync/tasks/worker.rake'
     end
+
+    configure do
+      require 'pub_sub_model_sync/initializers/before_commit' if PubSubModelSync::Config.enable_rails4_before_commit
+    end
   end
 end

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class RunSubscriber < Base
+    attr_accessor :subscriber, :payload, :model
+
+    delegate :settings, to: :subscriber
+
+    # @param subscriber(Subscriber)
+    # @param payload(Payload)
+    def initialize(subscriber, payload)
+      @subscriber = subscriber
+      @payload = payload
+    end
+
+    def call
+      klass_subscription? ? run_class_message : run_model_message
+    end
+
+    private
+
+    def klass_subscription?
+      subscriber.settings[:mode] == :klass
+    end
+
+    def run_class_message
+      model_class = subscriber.klass.constantize
+      model_class.ps_processing_payload = payload # TODO: review for parallel notifications
+      call_action(model_class) if ensure_sync(model_class)
+    end
+
+    # support for: create, update, destroy
+    def run_model_message
+      @model = find_model
+      model.ps_processing_payload = payload
+      return unless ensure_sync(model)
+
+      populate_model
+      model.send(:ps_before_save_sync) if model.respond_to?(:ps_before_save_sync)
+      call_action(model)
+    end
+
+    def ensure_sync(object)
+      res = true
+      res = false if settings[:if] && !parse_condition(settings[:if], object)
+      res = false if settings[:unless] && parse_condition(settings[:unless], object)
+      log("Cancelled save sync by subscriber condition : #{[payload]}") if !res && debug?
+      res
+    end
+
+    def call_action(object)
+      callback = settings[:to_action]
+      callback.is_a?(Proc) ? object.instance_exec(payload.data, &callback) : call_action_method(object)
+    end
+
+    def call_action_method(object)
+      method_name = settings[:to_action]
+      method_name = :save! if %i[create update].include?(method_name.to_sym)
+      method_name = :destroy! if method_name.to_sym == :destroy
+      is_crud_action = %i[save! destroy!].include?(method_name)
+      is_crud_action ? object.send(method_name) : object.send(method_name, payload.data)
+    end
+
+    def parse_condition(condition, object)
+      proc_args = klass_subscription? ? [] : [object]
+      case condition
+      when Proc then condition.call(*proc_args)
+      when Array then condition.all? { |method_name| object.send(method_name) }
+      else # method name
+        object.send(condition)
+      end
+    end
+
+    def find_model
+      model_class = subscriber.klass.constantize
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
+
+      error_msg = 'No values provided for identifiers:'
+      raise(StandardError, "#{error_msg} #{[settings[:id], payload]}") if model_identifiers.empty?
+
+      model_class.where(model_identifiers).first_or_initialize
+    end
+
+    # @param mappings (Array<String,Symbol>) supports aliasing, sample: ["id", "full_name:name"]
+    # @return (Hash) hash with the correct attr names and its values
+    def parse_mapping(mappings)
+      mappings.map do |prop|
+        source, target = prop.to_s.split(':')
+        key = (target || source).to_sym
+        next unless payload.data.key?(source.to_sym)
+
+        [key, payload.data[source.to_sym]]
+      end.compact.to_h.symbolize_keys
+    end
+
+    # @return (Hash) hash including identifiers and its values
+    def model_identifiers
+      @model_identifiers ||= parse_mapping(Array(settings[:id]))
+    end
+
+    def populate_model
+      values = parse_mapping(subscriber.mapping).except(model_identifiers.keys)
+      values.each do |attr, value|
+        model.send("#{attr}=", value)
+      end
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,50 +4,46 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
-    PUBLISH_SETTINGS = {}.freeze
-    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
-      raise 'method :listen_messages must be defined in service'
+      raise NoMethodError, 'method :listen_messages must be defined in service'
     end
 
     # @param _payload (Payload)
     def publish(_payload)
-      raise 'method :publish must be defined in service'
+      raise NoMethodError, 'method :publish must be defined in service'
     end
 
     def stop
-      raise 'method :stop must be defined in service'
+      raise NoMethodError, 'method :stop must be defined in service'
     end
 
     private
 
+    # @param payload (Payload)
+    # @return (String): Json Format
+    def encode_payload(payload)
+      data = payload.to_h
+      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      reduce_payload_size = !config.debug
+      data[:headers].except!(*not_important_keys) if reduce_payload_size
+      data.to_json
+    end
+
     # @param (String: Payload in json format)
     def process_message(payload_info)
-      retries ||= 0
-      payload = parse_payload(payload_info)
+      payload = decode_payload(payload_info)
       return payload.process unless same_app_message?(payload)
 
       log("Skipping message from same origin: #{[payload]}") if config.debug
     rescue => e
-      retry if can_retry_process_message?(e, payload, retries += 1)
-    end
-
-    def can_retry_process_message?(error, payload, retries)
-      error_payload = [payload, error.message, error.backtrace]
-      if retries == 1
-        log("Error while starting to process message (retrying...): #{error_payload}", :error)
-        rescue_database_connection if lost_db_connection_err?(error)
-      else
-        log("Retried 1 time and error persists, exiting...: #{error_payload}", :error)
-        Process.exit!(true)
-      end
-      retries == 1
+      error_payload = [payload, e.message, e.backtrace]
+      log("Error while starting to process a message: #{error_payload}", :error)
     end
 
-    def parse_payload(payload_info)
-      info = JSON.parse(payload_info).deep_symbolize_keys
-      payload = ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+    # @return Payload
+    def decode_payload(payload_info)
+      payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
       log("Received message: #{[payload]}") if config.debug
       payload
     end
@@ -57,19 +53,5 @@ module PubSubModelSync
       key = payload.headers[:app_key]
       key && key == config.subscription_key
     end
-
-    def lost_db_connection_err?(error)
-      return true if error.class.name == 'PG::UnableToSend' # rubocop:disable Style/ClassEqualityComparison
-
-      error.message.match?(/lost connection/i)
-    end
-
-    def rescue_database_connection
-      log('Lost DB connection. Attempting to reconnect...', :warn)
-      ActiveRecord::Base.connection.reconnect!
-    rescue
-      log('Cannot reconnect to database, exiting...', :error)
-      Process.exit!(true)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,50 +7,86 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true }.freeze
+    LISTEN_SETTINGS = { message_ordering: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
     TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
-    attr_accessor :service, :topic, :subscription, :subscriber
+
+    # @!attribute topics (Hash): { key: Topic1, ... }
+    # @!attribute publish_topics (Hash): { key: Topic1, ... }
+    attr_accessor :service, :topics, :subscribers, :publish_topics
 
     def initialize
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
-      @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name, TOPIC_SETTINGS)
-      topic.enable_message_ordering!
+      Array(config.topic_name || 'model_sync').each(&method(:init_topic))
     end
 
     def listen_messages
-      @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
-      subscriber.start
+      @subscribers = subscribe_to_topics
       log('Listener started')
       sleep
-      subscriber.stop.wait!
+      subscribers.each { |subscriber| subscriber.stop.wait! }
       log('Listener stopped')
     end
 
+    # @param payload (PubSubModelSync::Payload)
     def publish(payload)
-      topic.publish_async(payload.to_json, message_headers) do |res|
-        raise 'Failed to publish the message.' unless res.succeeded?
+      p_topic_names = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics = p_topic_names.map(&method(:find_topic))
+      message_topics.each do |topic|
+        topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
+          raise StandardError, 'Failed to publish the message.' unless res.succeeded?
+        end
       end
     end
 
     def stop
       log('Listener stopping...')
-      subscriber.stop!
+      subscribers.each(&:stop!)
     end
 
     private
 
-    def message_headers
-      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    def find_topic(topic_name)
+      topic_name = topic_name.to_s
+      return topics.values.first unless topic_name.present?
+
+      topics[topic_name] || publish_topics[topic_name] || init_topic(topic_name, only_publish: true)
     end
 
-    def subscribe_to_topic
-      topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
+    # @param only_publish (Boolean): if false is used to listen and publish messages
+    # @return (Topic): returns created or loaded topic
+    def init_topic(topic_name, only_publish: false)
+      topic_name = topic_name.to_s
+      @topics ||= {}
+      @publish_topics ||= {}
+      topic = service.topic(topic_name) || service.create_topic(topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
+      publish_topics[topic_name] = topic if only_publish
+      topics[topic_name] = topic unless only_publish
+      topic
+    end
+
+    # @param payload (PubSubModelSync::Payload)
+    def message_headers(payload)
+      {
+        SERVICE_KEY => true,
+        ordering_key: payload.headers[:ordering_key]
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    # @return [Subscriber]
+    def subscribe_to_topics
+      topics.map do |key, topic|
+        subs_name = "#{config.subscription_key}_#{key}"
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscriber.start
+        log("Subscribed to topic: #{topic.name} as: #{subs_name}")
+        subscriber
+      end
     end
 
     def process_message(received_message)