pub_sub_model_sync 0.5.9 → 1.0.beta1

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -2,73 +2,86 @@
 
 module PubSubModelSync
   module PublisherConcern
-    def self.included(base)
-      base.extend(ClassMethods)
-    end
+    extend ActiveSupport::Concern
 
-    # Before initializing sync service (callbacks: after create/update/destroy)
-    def ps_skip_callback?(_action)
-      false
+    included do
+      extend ClassMethods
+      ps_init_transaction_callbacks if self <= ActiveRecord::Base
     end
 
     # before preparing data to sync
-    def ps_skip_sync?(_action)
+    def ps_skip_publish?(_action)
       false
     end
+    alias ps_skip_sync? ps_skip_publish? # @deprecated
 
     # before delivering data (return :cancel to cancel sync)
-    def ps_before_sync(_action, _data); end
+    def ps_before_publish(_action, _payload); end
+    alias ps_before_sync ps_before_publish # @deprecated
 
     # after delivering data
-    def ps_after_sync(_action, _data); end
+    def ps_after_publish(_action, _payload); end
+    alias ps_after_sync ps_after_publish # @deprecated
 
-    # To perform sync on demand
-    # @param attrs (Array, optional): custom attrs to be used
-    # @param as_klass (Array, optional): custom klass name to be used
-    # @param publisher (Publisher, optional): custom publisher object
-    def ps_perform_sync(action = :create, attrs: nil, as_klass: nil,
-                        publisher: nil)
-      publisher ||= self.class.ps_publisher(action).dup
-      publisher.attrs = attrs if attrs
-      publisher.as_klass = as_klass if as_klass
-      PubSubModelSync::MessagePublisher.publish_model(self, action, publisher)
+    # Delivers a notification via pubsub
+    # @param action (Sym|String) Sample: create|update|save|destroy|<any_other_key>
+    # @param mapping? (Array<String>) If present will generate data using the mapping and added to the payload.
+    #   Sample: ["id", "full_name:name"]
+    # @param data? (Hash|Symbol|Proc)
+    #   Hash: Data to be added to the payload
+    #   Symbol: Method name to be called to retrieve payload data (must return a hash value, receives :action name)
+    #   Proc: Block to be called to retrieve payload data
+    # @param headers? (Hash|Symbol|Proc): (All available attributes in Payload.headers)
+    #   Hash: Data that will be merged with default header values
+    #   Symbol: Method name that will be called to retrieve header values (must return a hash, receives :action name)
+    #   Proc: Block to be called to retrieve header values
+    # @param as_klass? (String): Output class name used instead of current class name
+    def ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: self.class.name)
+      p_klass = PubSubModelSync::MessagePublisher
+      p_klass.publish_model(self, action, data: data, mapping: mapping, headers: headers, as_klass: as_klass)
     end
+    delegate :ps_class_publish, to: :class
 
     module ClassMethods
-      # Permit to configure to publish crud actions (:create, :update, :destroy)
-      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil)
-        klass = PubSubModelSync::Publisher
-        publisher = klass.new(attrs, name, actions, as_klass)
-        PubSubModelSync::Config.publishers << publisher
-        actions.each do |action|
-          ps_register_callback(action.to_sym, publisher)
-        end
-      end
-
-      # On demand class level publisher
-      def ps_class_publish(data, action:, as_klass: nil)
-        as_klass = (as_klass || name).to_s
+      # Publishes a class level notification via pubsub
+      # @param data (Hash): Data of the notification
+      # @param action (Symbol): action  name of the notification
+      # @param as_klass (String, default current class name): Class name of the notification
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      def ps_class_publish(data, action:, as_klass: nil, headers: {})
         klass = PubSubModelSync::MessagePublisher
-        klass.publish_data(as_klass, data, action.to_sym)
+        klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
       end
 
-      # Publisher info for specific action
-      def ps_publisher(action = :create)
-        PubSubModelSync::Config.publishers.find do |publisher|
-          publisher.klass == name && publisher.actions.include?(action)
+      # @param crud_actions (Symbol|Array<Symbol>): :create, :update, :destroy
+      # @param method_name (Symbol, optional) method to be called
+      def ps_on_crud_event(crud_actions, method_name = nil, &block) # rubocop:disable Metrics/MethodLength
+        callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
+        commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
+        Array(crud_actions).each do |action|
+          if action == :destroy
+            after_destroy { instance_exec(action, &callback) }
+          elsif PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
+            define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
+          else
+            send(commit_name, on: action) { instance_exec(action, &callback) }
+          end
         end
       end
 
       private
 
-      def ps_register_callback(action, publisher)
-        after_commit(on: action) do |model|
-          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
-          if !disabled && !model.ps_skip_callback?(action)
-            klass = PubSubModelSync::MessagePublisher
-            klass.publish_model(model, action.to_sym, publisher)
-          end
+      # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
+      def ps_init_transaction_callbacks
+        start_transaction = lambda do
+          key = id ? PubSubModelSync::Publisher.ordering_key_for(self) : nil
+          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(key)
         end
+        after_create start_transaction, prepend: true # wait for ID
+        before_update start_transaction, prepend: true
+        before_destroy start_transaction, prepend: true
+        after_commit { @ps_transaction.finish }
+        after_rollback(prepend: true) { @ps_transaction.rollback }
       end
     end
   end

data/lib/pub_sub_model_sync/railtie.rb

@@ -2,6 +2,7 @@
 
 require 'pub_sub_model_sync'
 require 'rails'
+require 'pub_sub_model_sync/config'
 module PubSubModelSync
   class Railtie < ::Rails::Railtie
     railtie_name :pub_sub_model_sync
@@ -9,5 +10,9 @@ module PubSubModelSync
     rake_tasks do
       load 'pub_sub_model_sync/tasks/worker.rake'
     end
+
+    configure do
+      require 'pub_sub_model_sync/initializers/before_commit' if PubSubModelSync::Config.enable_rails4_before_commit
+    end
   end
 end

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class RunSubscriber < Base
+    attr_accessor :subscriber, :payload, :model
+
+    delegate :settings, to: :subscriber
+
+    # @param subscriber(Subscriber)
+    # @param payload(Payload)
+    def initialize(subscriber, payload)
+      @subscriber = subscriber
+      @payload = payload
+    end
+
+    def call
+      klass_subscription? ? run_class_message : run_model_message
+    end
+
+    private
+
+    def klass_subscription?
+      subscriber.settings[:mode] == :klass
+    end
+
+    def run_class_message
+      model_class = subscriber.klass.constantize
+      model_class.ps_processing_payload = payload # TODO: review for parallel notifications
+      call_action(model_class, payload.data) if ensure_sync(model_class)
+    end
+
+    # support for: create, update, destroy
+    def run_model_message
+      @model = find_model
+      model.ps_processing_payload = payload
+      return unless ensure_sync(model)
+
+      populate_model
+      model.send(:ps_before_save_sync) if model.respond_to?(:ps_before_save_sync)
+      call_action(model)
+    end
+
+    def ensure_sync(object)
+      res = true
+      res = false if settings[:if] && !parse_condition(settings[:if], object)
+      res = false if settings[:unless] && parse_condition(settings[:unless], object)
+      log("Cancelled save sync by subscriber condition : #{[payload]}") if !res && debug?
+      res
+    end
+
+    def call_action(object, *args)
+      action_name = settings[:to_action]
+      if action_name.is_a?(Proc)
+        args.prepend(object) unless klass_subscription?
+        action_name.call(*args)
+      else # method name
+        action_name = :save if %i[create update].include?(action_name.to_sym)
+        object.send(action_name, *args)
+      end
+      raise(object.errors) if object.respond_to?(:errors) && object.errors.any?
+    end
+
+    def parse_condition(condition, object)
+      proc_args = klass_subscription? ? [] : [object]
+      case condition
+      when Proc then condition.call(*proc_args)
+      when Array then condition.all? { |method_name| object.send(method_name) }
+      else # method name
+        object.send(condition)
+      end
+    end
+
+    def find_model
+      model_class = subscriber.klass.constantize
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
+
+      model_class.where(model_identifiers).first_or_initialize
+    end
+
+    # @param mappings (Array<String>) supports aliasing, sample: ["id", "full_name:name"]
+    # @return (Hash) hash with the correct attr names and its values
+    def parse_mapping(mappings)
+      mappings.map do |prop|
+        source, target = prop.to_s.split(':')
+        key = (target || source).to_sym
+        next unless payload.data.key?(source.to_sym)
+
+        [key, payload.data[source.to_sym]]
+      end.compact.to_h.symbolize_keys
+    end
+
+    # @return (Hash) hash including identifiers and its values
+    def model_identifiers
+      @model_identifiers ||= parse_mapping(Array(settings[:id]).map(&:to_s))
+    end
+
+    def populate_model
+      values = parse_mapping(subscriber.mapping).except(model_identifiers.keys)
+      values.each do |attr, value|
+        model.send("#{attr}=", value)
+      end
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,8 +4,6 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
-    PUBLISH_SETTINGS = {}.freeze
-    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'
@@ -22,23 +20,45 @@ module PubSubModelSync
 
     private
 
+    # @param payload (Payload)
+    # @return (String): Json Format
+    def encode_payload(payload)
+      data = payload.to_h
+      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      reduce_payload_size = !config.debug
+      data[:headers].except!(*not_important_keys) if reduce_payload_size
+      data.to_json
+    end
+
     # @param (String: Payload in json format)
     def process_message(payload_info)
-      payload = parse_payload(payload_info)
-      log("Received message: #{[payload]}") if config.debug
-      if same_app_message?(payload)
-        log("Skip message from same origin: #{[payload]}") if config.debug
+      retries ||= 0
+      payload = decode_payload(payload_info)
+      return payload.process unless same_app_message?(payload)
+
+      log("Skipping message from same origin: #{[payload]}") if config.debug
+    rescue => e
+      retry if can_retry_process_message?(e, payload, retries += 1)
+    end
+
+    def can_retry_process_message?(error, payload, retries)
+      error_payload = [payload, error.message, error.backtrace]
+      if retries <= 5
+        sleep(retries)
+        log("Error while starting to process a message (retrying #{retries} retries...): #{error_payload}", :error)
+        rescue_database_connection if lost_db_connection_err?(error)
+        true
       else
-        payload.process
+        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
+        Process.exit!(true)
       end
-    rescue => e
-      error = [payload, e.message, e.backtrace]
-      log("Error parsing received message: #{error}", :error)
     end
 
-    def parse_payload(payload_info)
-      info = JSON.parse(payload_info).deep_symbolize_keys
-      ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+    # @return Payload
+    def decode_payload(payload_info)
+      payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
+      log("Received message: #{[payload]}") if config.debug
+      payload
     end
 
     # @param payload (Payload)
@@ -46,5 +66,19 @@ module PubSubModelSync
       key = payload.headers[:app_key]
       key && key == config.subscription_key
     end
+
+    def lost_db_connection_err?(error)
+      return true if error.class.name == 'PG::UnableToSend' # rubocop:disable Style/ClassEqualityComparison
+
+      error.message.match?(/lost connection/i)
+    end
+
+    def rescue_database_connection
+      log('Lost DB connection. Attempting to reconnect...', :warn)
+      ActiveRecord::Base.connection.reconnect!
+    rescue
+      log('Cannot reconnect to database, exiting...', :error)
+      Process.exit!(true)
+    end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,50 +7,86 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true }.freeze
+    LISTEN_SETTINGS = { message_ordering: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
     TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
-    attr_accessor :service, :topic, :subscription, :subscriber
+
+    # @!attribute topics (Hash): { key: Topic1, ... }
+    # @!attribute publish_topics (Hash): { key: Topic1, ... }
+    attr_accessor :service, :topics, :subscribers, :publish_topics
 
     def initialize
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
-      @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name, TOPIC_SETTINGS)
-      topic.enable_message_ordering!
+      Array(config.topic_name || 'model_sync').each(&method(:init_topic))
     end
 
     def listen_messages
-      @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
-      subscriber.start
+      @subscribers = subscribe_to_topics
       log('Listener started')
       sleep
-      subscriber.stop.wait!
+      subscribers.each { |subscriber| subscriber.stop.wait! }
       log('Listener stopped')
     end
 
+    # @param payload (PubSubModelSync::Payload)
     def publish(payload)
-      topic.publish_async(payload.to_json, message_headers) do |res|
-        raise 'Failed to publish the message.' unless res.succeeded?
+      p_topic_names = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics = p_topic_names.map(&method(:find_topic))
+      message_topics.each do |topic|
+        topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
+          raise 'Failed to publish the message.' unless res.succeeded?
+        end
       end
     end
 
     def stop
       log('Listener stopping...')
-      subscriber.stop!
+      subscribers.each(&:stop!)
     end
 
     private
 
-    def message_headers
-      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    def find_topic(topic_name)
+      topic_name = topic_name.to_s
+      return topics.values.first unless topic_name.present?
+
+      topics[topic_name] || publish_topics[topic_name] || init_topic(topic_name, only_publish: true)
     end
 
-    def subscribe_to_topic
-      topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
+    # @param only_publish (Boolean): if false is used to listen and publish messages
+    # @return (Topic): returns created or loaded topic
+    def init_topic(topic_name, only_publish: false)
+      topic_name = topic_name.to_s
+      @topics ||= {}
+      @publish_topics ||= {}
+      topic = service.topic(topic_name) || service.create_topic(topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
+      publish_topics[topic_name] = topic if only_publish
+      topics[topic_name] = topic unless only_publish
+      topic
+    end
+
+    # @param payload (PubSubModelSync::Payload)
+    def message_headers(payload)
+      {
+        SERVICE_KEY => true,
+        ordering_key: payload.headers[:ordering_key]
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    # @return [Subscriber]
+    def subscribe_to_topics
+      topics.map do |key, topic|
+        subs_name = "#{config.subscription_key}_#{key}"
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscriber.start
+        log("Subscribed to topic: #{topic.name} as: #{subs_name}")
+        subscriber
+      end
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -7,14 +7,20 @@ end
 
 module PubSubModelSync
   class ServiceKafka < ServiceBase
+    QTY_WORKERS = 10
+    LISTEN_SETTINGS = {}.freeze
+    PUBLISH_SETTINGS = {}.freeze
+    PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
     cattr_accessor :producer
-    attr_accessor :config, :service, :consumer
+
+    # @!attribute topic_names (Array): ['topic 1', 'topic 2']
+    attr_accessor :service, :consumer, :topic_names
 
     def initialize
-      @config = PubSubModelSync::Config
       settings = config.kafka_connection
       settings[1][:client_id] ||= config.subscription_key
       @service = Kafka.new(*settings)
+      @topic_names = ensure_topics(Array(config.topic_name || 'model_sync'))
     end
 
     def listen_messages
@@ -28,12 +34,10 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      settings = {
-        topic: config.topic_name,
-        headers: { SERVICE_KEY => true }
-      }.merge(PUBLISH_SETTINGS)
-      producer.produce(payload.to_json, settings)
-      producer.deliver_messages
+      message_topics = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics.each do |topic_name|
+        producer.produce(encode_payload(payload), message_settings(payload, topic_name))
+      end
     end
 
     def stop
@@ -43,22 +47,45 @@ module PubSubModelSync
 
     private
 
+    def message_settings(payload, topic_name)
+      {
+        topic: ensure_topics(topic_name),
+        partition_key: payload.headers[:ordering_key],
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+    end
+
     def start_consumer
-      @consumer = service.consumer(group_id: config.subscription_key)
-      consumer.subscribe(config.topic_name)
+      subscription_key = config.subscription_key
+      @consumer = service.consumer(group_id: subscription_key)
+      topic_names.each do |topic_name|
+        log("Subscribed to topic: #{topic_name} as #{subscription_key}")
+        consumer.subscribe(topic_name)
+      end
     end
 
     def producer
       return self.class.producer if self.class.producer
 
       at_exit { self.class.producer.shutdown }
-      self.class.producer = service.producer
+      self.class.producer = service.async_producer(PRODUCER_SETTINGS)
     end
 
     def process_message(message)
-      return unless message.headers[SERVICE_KEY]
+      super(message.value) if message.headers[SERVICE_KEY]
+    end
 
-      super(message.value)
+    # Check topic existence, create if missing topic
+    # @param names (Array<String>|String)
+    # @return (Array|String) return @param names
+    def ensure_topics(names)
+      missing_topics = Array(names) - (@known_topics || service.topics)
+      missing_topics.each do |name|
+        service.create_topic(name)
+      end
+      @known_topics ||= [] # cache service.topics to reduce verification time
+      @known_topics = (@known_topics + Array(names)).uniq
+      names
     end
   end
 end