pub_sub_model_sync 0.4.0 → 0.5.0.1

data/gemfiles/Gemfile_5

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem 'rubocop'
+gem 'bunny' # rabbit-mq
+gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
+gem 'rails', '~> 5'
+
+group :test do
+  gem 'database_cleaner-active_record'
+end
+
+# Specify your gem's dependencies in pub_sub_model_sync.gemspec
+gemspec

data/gemfiles/Gemfile_6

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem 'rubocop'
+gem 'bunny' # rabbit-mq
+gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
+gem 'rails', '~> 6'
+
+group :test do
+  gem 'database_cleaner-active_record'
+end
+
+# Specify your gem's dependencies in pub_sub_model_sync.gemspec
+gemspec

data/lib/pub_sub_model_sync.rb

@@ -5,6 +5,7 @@ require 'active_support'
 
 require 'pub_sub_model_sync/railtie'
 require 'pub_sub_model_sync/config'
+require 'pub_sub_model_sync/base'
 require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'

data/lib/pub_sub_model_sync/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Base
+    delegate :config, :log, to: self
+
+    class << self
+      def config
+        PubSubModelSync::Config
+      end
+
+      def log(message, kind = :info)
+        config.log message, kind
+      end
+    end
+  end
+end

data/lib/pub_sub_model_sync/config.rb

@@ -5,16 +5,26 @@ module PubSubModelSync
     cattr_accessor(:subscribers) { [] }
     cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
-    cattr_accessor :logger
+
+    # customizable callbacks
+    cattr_accessor(:debug) { false }
+    cattr_accessor :logger # LoggerInst
+
+    cattr_accessor(:on_process_success) { ->(_payload, _subscriber) {} }
+    cattr_accessor(:on_process_error) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_before_publish) { ->(_payload) {} }
+    cattr_accessor(:on_after_publish) { ->(_payload) {} }
+    cattr_accessor(:on_publish_error) { ->(_exception, _payload) {} }
+    cattr_accessor(:disabled) { false }
 
     # google service
     cattr_accessor :project, :credentials, :topic_name, :subscription_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name
+    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
@@ -24,5 +34,10 @@ module PubSubModelSync
         logger ? logger.send(kind, msg) : puts(msg)
       end
     end
+
+    def self.subscription_key
+      subscription_name ||
+        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+    end
   end
 end

data/lib/pub_sub_model_sync/connector.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Connector
     attr_accessor :service
+
     delegate :listen_messages, :publish, :stop, to: :service
 
     def initialize

data/lib/pub_sub_model_sync/message_processor.rb

@@ -1,40 +1,46 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class MessageProcessor
-    attr_accessor :data, :klass, :action
-
-    # @param data (Hash): any hash value to deliver
-    def initialize(data, klass, action)
-      @data = data
-      @klass = klass
-      @action = action
+  class MessageProcessor < PubSubModelSync::Base
+    attr_accessor :payload
+
+    # @param payload (Payload): payload to be delivered
+    # @Deprecated: def initialize(data, klass, action)
+    def initialize(payload, klass = nil, action = nil)
+      @payload = payload
+      return if @payload.is_a?(Payload)
+
+      # support for deprecated
+      log('Deprecated: Use Payload instead of new(data, klass, action)')
+      @payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
     end
 
     def process
-      subscribers = filter_subscribers
-      subscribers.each { |subscriber| run_subscriber(subscriber) }
+      filter_subscribers.each(&method(:run_subscriber))
     end
 
     private
 
     def run_subscriber(subscriber)
-      subscriber.eval_message(data)
-      log "processed message with: #{[klass, action, data]}"
+      subscriber.eval_message(payload.data)
+      config.on_process_success.call(payload, subscriber)
+      log "processed message with: #{payload}"
     rescue => e
-      info = [klass, action, data, e.message, e.backtrace]
-      log("error processing message: #{info}", :error)
+      print_subscriber_error(e)
     end
 
-    def filter_subscribers
-      PubSubModelSync::Config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == klass.to_s &&
-          subscriber.settings[:from_action].to_s == action.to_s
-      end
+    # @param error (Error)
+    def print_subscriber_error(error)
+      info = [payload, error.message, error.backtrace]
+      res = config.on_process_error.call(error, payload)
+      log("Error processing message: #{info}", :error) if res != :skip_log
     end
 
-    def log(message, kind = :info)
-      PubSubModelSync::Config.log message, kind
+    def filter_subscribers
+      config.subscribers.select do |subscriber|
+        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
+          subscriber.settings[:from_action].to_s == payload.action.to_s
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -1,17 +1,15 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class MessagePublisher
+  class MessagePublisher < PubSubModelSync::Base
     class << self
-      delegate :publish, to: :connector
-
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
       def publish_data(klass, data, action)
-        attrs = { klass: klass.to_s, action: action.to_sym }
-        publish(data, attrs)
+        payload = PubSubModelSync::Payload.new(data, { klass: klass, action: action.to_sym })
+        publish(payload)
       end
 
       # @param model: ActiveRecord model
@@ -21,12 +19,30 @@ module PubSubModelSync
         return if model.ps_skip_sync?(action)
 
         publisher ||= model.class.ps_publisher(action)
-        payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload[:data])
+        payload_info = publisher.payload(model, action)
+        payload = PubSubModelSync::Payload.new(payload_info[:data], payload_info[:attrs])
+        res_before = model.ps_before_sync(action, payload.data)
         return if res_before == :cancel
 
-        publish(payload[:data], payload[:attrs])
-        model.ps_after_sync(action, payload[:data])
+        publish(payload)
+        model.ps_after_sync(action, payload.data)
+      end
+
+      def publish(payload)
+        log("Publishing message: #{[payload]}") if config.debug
+        config.on_before_publish.call(payload)
+        connector.publish(payload)
+        config.on_after_publish.call(payload)
+      rescue => e
+        notify_error(e, payload)
+      end
+
+      private
+
+      def notify_error(exception, payload)
+        info = [payload, exception.message, exception.backtrace]
+        res = config.on_publish_error.call(exception, payload)
+        log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end
   end

data/lib/pub_sub_model_sync/mock_rabbit_service.rb

@@ -20,12 +20,17 @@ module PubSubModelSync
       def name
         'name'
       end
+
+      def publish(*_args)
+        true
+      end
     end
 
     class MockChannel
       def queue(*_args)
         @queue ||= MockQueue.new
       end
+      alias fanout queue
 
       def topic(*_args)
         @topic ||= MockTopic.new

data/lib/pub_sub_model_sync/payload.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Payload
+    attr_reader :data, :attributes, :headers
+
+    # @param data (Hash: { any value }):
+    # @param attributes (Hash: { klass: string, action: :sym }):
+    def initialize(data, attributes, headers = {})
+      @data = data
+      @attributes = attributes
+      @headers = headers
+      build_headers
+    end
+
+    def to_h
+      { data: data, attributes: attributes, headers: headers }
+    end
+
+    def klass
+      attributes[:klass]
+    end
+
+    def action
+      attributes[:action]
+    end
+
+    def process!
+      publisher = PubSubModelSync::MessageProcessor.new(self)
+      publisher.process
+    end
+
+    def publish!
+      klass = PubSubModelSync::MessagePublisher
+      klass.publish(self)
+    end
+
+    private
+
+    def build_headers
+      headers[:uuid] ||= SecureRandom.uuid
+      headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+    end
+  end
+end

data/lib/pub_sub_model_sync/publisher.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Publisher
     attr_accessor :attrs, :actions, :klass, :as_klass
+
     def initialize(attrs, klass, actions = nil, as_klass = nil)
       @attrs = attrs
       @klass = klass

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -11,6 +11,7 @@ module PubSubModelSync
       false
     end
 
+    # TODO: make it using respond_to?(:ps_skip_sync?)
     # before preparing data to sync
     def ps_skip_sync?(_action)
       false
@@ -63,7 +64,8 @@ module PubSubModelSync
 
       def ps_register_callback(action, publisher)
         after_commit(on: action) do |model|
-          unless model.ps_skip_callback?(action)
+          disabled = PubSubModelSync::Config.disabled
+          if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
             klass.publish_model(model, action.to_sym, publisher)
           end

data/lib/pub_sub_model_sync/service_base.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
+require 'pub_sub_model_sync/payload'
 module PubSubModelSync
-  class ServiceBase
+  class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'
     end
 
-    def publish(_data, _attributes)
+    # @param _payload (Payload)
+    def publish(_payload)
       raise 'method :publish must be defined in service'
     end
 
@@ -18,19 +20,29 @@ module PubSubModelSync
 
     private
 
-    # @param payload (String JSON): '{"data":{}, "attributes":{..}}'
-    #   refer: PubSubModelSync::MessagePublisher(.publish_model | .publish_data)
-    def perform_message(payload)
-      data, attrs = parse_message_payload(payload)
-      args = [data, attrs[:klass], attrs[:action]]
-      PubSubModelSync::MessageProcessor.new(*args).process
+    # @param (String: Payload in json format)
+    def process_message(payload_info)
+      payload = parse_payload(payload_info)
+      log("Received message: #{[payload]}") if config.debug
+      if same_app_message?(payload)
+        log("Skip message from same origin: #{[payload]}") if config.debug
+      else
+        payload.process!
+      end
+    rescue => e
+      error = [payload, e.message, e.backtrace]
+      log("Error parsing received message: #{error}", :error)
     end
 
-    def parse_message_payload(payload)
-      message_payload = JSON.parse(payload).symbolize_keys
-      data = message_payload[:data].symbolize_keys
-      attrs = message_payload[:attributes].symbolize_keys
-      [data, attrs]
+    def parse_payload(payload_info)
+      info = JSON.parse(payload_info).deep_symbolize_keys
+      ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+    end
+
+    # @param payload (Payload)
+    def same_app_message?(payload)
+      key = payload.headers[:app_key]
+      key && key == config.subscription_key
     end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,10 +7,9 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    attr_accessor :service, :topic, :subscription, :config, :subscriber
+    attr_accessor :service, :topic, :subscription, :subscriber
 
     def initialize
-      @config = PubSubModelSync::Config
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
       @topic = service.topic(config.topic_name) ||
@@ -28,13 +27,8 @@ module PubSubModelSync
       log('Listener stopped')
     end
 
-    def publish(data, attributes)
-      log("Publishing message: #{[data, attributes]}")
-      payload = { data: data, attributes: attributes }.to_json
-      topic.publish(payload, { SERVICE_KEY => true })
-    rescue => e
-      info = [data, attributes, e.message, e.backtrace]
-      log("Error publishing: #{info}", :error)
+    def publish(payload)
+      topic.publish(payload.to_json, { SERVICE_KEY => true })
     end
 
     def stop
@@ -51,17 +45,9 @@ module PubSubModelSync
 
     def process_message(received_message)
       message = received_message.message
-      return unless message.attributes[SERVICE_KEY]
-
-      perform_message(message.data)
-    rescue => e
-      log("Error processing message: #{[received_message, e.message]}", :error)
+      super(message.data) if message.attributes[SERVICE_KEY]
     ensure
       received_message.acknowledge!
     end
-
-    def log(msg, kind = :info)
-      config.log("Google Service ==> #{msg}", kind)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -8,9 +8,8 @@ end
 module PubSubModelSync
   class ServiceKafka < ServiceBase
     cattr_accessor :producer
+    attr_accessor :config, :service, :consumer
 
-    attr_accessor :service, :consumer
-    attr_accessor :config
     CONSUMER_GROUP = 'service_model_sync'
 
     def initialize
@@ -23,19 +22,14 @@ module PubSubModelSync
       start_consumer
       consumer.each_message(&method(:process_message))
     rescue PubSubModelSync::Runner::ShutDown
-      raise
+      log('Listener stopped')
     rescue => e
       log("Error listening message: #{[e.message, e.backtrace]}", :error)
     end
 
-    def publish(data, attributes)
-      log("Publishing: #{[data, attributes]}")
-      payload = { data: data, attributes: attributes }
+    def publish(payload)
       producer.produce(payload.to_json, message_settings)
       producer.deliver_messages
-    rescue => e
-      info = [data, attributes, e.message, e.backtrace]
-      log("Error publishing: #{info}", :error)
     end
 
     def stop
@@ -64,14 +58,7 @@ module PubSubModelSync
     def process_message(message)
       return unless message.headers[SERVICE_KEY]
 
-      perform_message(message.value)
-    rescue => e
-      error = [message, e.message, e.backtrace]
-      log("Error processing message: #{error}", :error)
-    end
-
-    def log(msg, kind = :info)
-      config.log("Kafka Service ==> #{msg}", kind)
+      super(message.value)
     end
   end
 end