pub_sub_model_sync 0.5.8.1 → 0.6.0

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,8 +4,6 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
-    PUBLISH_SETTINGS = {}.freeze
-    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'
@@ -22,23 +20,45 @@ module PubSubModelSync
 
     private
 
+    # @param payload (Payload)
+    # @return (String): Json Format
+    def encode_payload(payload)
+      data = payload.to_h
+      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      reduce_payload_size = !config.debug
+      data[:headers].except!(*not_important_keys) if reduce_payload_size
+      data.to_json
+    end
+
     # @param (String: Payload in json format)
     def process_message(payload_info)
-      payload = parse_payload(payload_info)
-      log("Received message: #{[payload]}") if config.debug
-      if same_app_message?(payload)
-        log("Skip message from same origin: #{[payload]}") if config.debug
+      retries ||= 0
+      payload = decode_payload(payload_info)
+      return payload.process unless same_app_message?(payload)
+
+      log("Skipping message from same origin: #{[payload]}") if config.debug
+    rescue => e
+      retry if can_retry_process_message?(e, payload, retries += 1)
+    end
+
+    def can_retry_process_message?(error, payload, retries)
+      error_payload = [payload, error.message, error.backtrace]
+      if retries == 1
+        log("Error while starting to process message (retrying...): #{error_payload}", :error)
+        rescue_database_connection if lost_db_connection_err?(error)
       else
-        payload.process
+        log("Retried 1 time and error persists, exiting...: #{error_payload}", :error)
+        Process.exit!(true)
       end
-    rescue => e
-      error = [payload, e.message, e.backtrace]
-      log("Error parsing received message: #{error}", :error)
+      retries == 1
     end
 
-    def parse_payload(payload_info)
+    # @return Payload
+    def decode_payload(payload_info)
       info = JSON.parse(payload_info).deep_symbolize_keys
-      ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+      payload = ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+      log("Received message: #{[payload]}") if config.debug
+      payload
     end
 
     # @param payload (Payload)
@@ -46,5 +66,19 @@ module PubSubModelSync
       key = payload.headers[:app_key]
       key && key == config.subscription_key
     end
+
+    def lost_db_connection_err?(error)
+      return true if error.class.name == 'PG::UnableToSend' # rubocop:disable Style/ClassEqualityComparison
+
+      error.message.match?(/lost connection/i)
+    end
+
+    def rescue_database_connection
+      log('Lost DB connection. Attempting to reconnect...', :warn)
+      ActiveRecord::Base.connection.reconnect!
+    rescue
+      log('Cannot reconnect to database, exiting...', :error)
+      Process.exit!(true)
+    end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,50 +7,85 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true, streams: 1 }.freeze
-    TOPIC_SETTINGS = { async: { threads: { publish: 1, callback: 1 } } }.freeze
+    LISTEN_SETTINGS = { message_ordering: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
+    TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
-    attr_accessor :service, :topic, :subscription, :subscriber
+
+    # @!attribute topics (Hash): { key: Topic1, ... }
+    # @!attribute publish_topics (Hash): { key: Topic1, ... }
+    attr_accessor :service, :topics, :subscribers, :publish_topics
 
     def initialize
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
-      @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name, TOPIC_SETTINGS)
-      topic.enable_message_ordering!
+      Array(config.topic_name || 'model_sync').each(&method(:init_topic))
     end
 
     def listen_messages
-      @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
-      subscriber.start
+      @subscribers = subscribe_to_topics
       log('Listener started')
       sleep
-      subscriber.stop.wait!
+      subscribers.each { |subscriber| subscriber.stop.wait! }
       log('Listener stopped')
     end
 
+    # @param payload (PubSubModelSync::Payload)
     def publish(payload)
-      topic.publish_async(payload.to_json, message_headers) do |res|
-        raise 'Failed to publish the message.' unless res.succeeded?
+      message_topics = Array(payload.headers[:topic_name] || '').map(&method(:find_topic))
+      message_topics.each do |topic|
+        topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
+          raise 'Failed to publish the message.' unless res.succeeded?
+        end
       end
     end
 
     def stop
       log('Listener stopping...')
-      subscriber.stop!
+      subscribers.each(&:stop!)
     end
 
     private
 
-    def message_headers
-      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    def find_topic(topic_name)
+      topic_name = topic_name.to_s
+      return topics.values.first unless topic_name.present?
+
+      topics[topic_name] || publish_topics[topic_name] || init_topic(topic_name, only_publish: true)
     end
 
-    def subscribe_to_topic
-      topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
+    # @param only_publish (Boolean): if false is used to listen and publish messages
+    # @return (Topic): returns created or loaded topic
+    def init_topic(topic_name, only_publish: false)
+      topic_name = topic_name.to_s
+      @topics ||= {}
+      @publish_topics ||= {}
+      topic = service.topic(topic_name) || service.create_topic(topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
+      publish_topics[topic_name] = topic if only_publish
+      topics[topic_name] = topic unless only_publish
+      topic
+    end
+
+    # @param payload (PubSubModelSync::Payload)
+    def message_headers(payload)
+      {
+        SERVICE_KEY => true,
+        ordering_key: payload.headers[:ordering_key]
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    # @return [Subscriber]
+    def subscribe_to_topics
+      topics.map do |key, topic|
+        subs_name = "#{config.subscription_key}_#{key}"
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscriber.start
+        log("Subscribed to topic: #{topic.name} as: #{subs_name}")
+        subscriber
+      end
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -7,14 +7,20 @@ end
 
 module PubSubModelSync
   class ServiceKafka < ServiceBase
+    QTY_WORKERS = 10
+    LISTEN_SETTINGS = {}.freeze
+    PUBLISH_SETTINGS = {}.freeze
+    PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
     cattr_accessor :producer
-    attr_accessor :config, :service, :consumer
+
+    # @!attribute topic_names (Array): ['topic 1', 'topic 2']
+    attr_accessor :service, :consumer, :topic_names
 
     def initialize
-      @config = PubSubModelSync::Config
       settings = config.kafka_connection
       settings[1][:client_id] ||= config.subscription_key
       @service = Kafka.new(*settings)
+      @topic_names = ensure_topics(Array(config.topic_name || 'model_sync'))
     end
 
     def listen_messages
@@ -28,12 +34,10 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      settings = {
-        topic: config.topic_name,
-        headers: { SERVICE_KEY => true }
-      }.merge(PUBLISH_SETTINGS)
-      producer.produce(payload.to_json, settings)
-      producer.deliver_messages
+      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics.each do |topic_name|
+        producer.produce(encode_payload(payload), message_settings(payload, topic_name))
+      end
     end
 
     def stop
@@ -43,22 +47,41 @@ module PubSubModelSync
 
     private
 
+    def message_settings(payload, topic_name)
+      {
+        topic: ensure_topics(topic_name),
+        partition_key: payload.headers[:ordering_key],
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+    end
+
     def start_consumer
       @consumer = service.consumer(group_id: config.subscription_key)
-      consumer.subscribe(config.topic_name)
+      topic_names.each { |topic_name| consumer.subscribe(topic_name) }
     end
 
     def producer
       return self.class.producer if self.class.producer
 
       at_exit { self.class.producer.shutdown }
-      self.class.producer = service.producer
+      self.class.producer = service.async_producer(PRODUCER_SETTINGS)
     end
 
     def process_message(message)
-      return unless message.headers[SERVICE_KEY]
+      super(message.value) if message.headers[SERVICE_KEY]
+    end
 
-      super(message.value)
+    # Check topic existence, create if missing topic
+    # @param names (Array<String>|String)
+    # @return (Array|String) return @param names
+    def ensure_topics(names)
+      missing_topics = Array(names) - (@known_topics || service.topics)
+      missing_topics.each do |name|
+        service.create_topic(name)
+      end
+      @known_topics ||= [] # cache service.topics to reduce verification time
+      @known_topics = (@known_topics + Array(names)).uniq
+      names
     end
   end
 end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -7,18 +7,26 @@ end
 
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
-    attr_accessor :config, :service, :channel, :queue, :topic
+    QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
+    LISTEN_SETTINGS = { manual_ack: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
+
+    # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
+    # @!attribute channels (Array): [Channel1]
+    # @!attribute exchanges (Hash<key: Exchange>): {topic_name: Exchange1}
+    attr_accessor :service, :topic_names, :channels, :exchanges
 
     def initialize
-      @config = PubSubModelSync::Config
       @service = Bunny.new(*config.bunny_connection)
+      @topic_names = Array(config.topic_name || 'model_sync')
+      @channels = []
+      @exchanges = {}
     end
 
     def listen_messages
       log('Listener starting...')
-      subscribe_to_queue
+      subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
       log('Listener started')
-      queue.subscribe(subscribe_settings, &method(:process_message))
       loop { sleep 5 }
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
@@ -40,54 +48,53 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
-      channel&.close
+      channels.each(&:close)
       service.close
     end
 
     private
 
-    def message_settings
+    def message_settings(payload)
       {
-        routing_key: queue.name,
+        routing_key: payload.headers[:ordering_key],
         type: SERVICE_KEY,
         persistent: true
       }.merge(PUBLISH_SETTINGS)
     end
 
-    def queue_settings
-      { durable: true, auto_delete: false }
-    end
-
-    def subscribe_settings
-      { manual_ack: false }.merge(LISTEN_SETTINGS)
-    end
-
     def process_message(_delivery_info, meta_info, payload)
-      return unless meta_info[:type] == SERVICE_KEY
-
-      super(payload)
+      super(payload) if meta_info[:type] == SERVICE_KEY
     end
 
-    def subscribe_to_queue
-      service.start
-      @channel = service.create_channel
-      @queue = channel.queue(config.subscription_key, queue_settings)
-      subscribe_to_exchange
+    def subscribe_to_queues(&block)
+      @channels = []
+      topic_names.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |channel, exchange|
+          queue = channel.queue(config.subscription_key, QUEUE_SETTINGS)
+          queue.bind(exchange)
+          @channels << channel
+          block.call(queue)
+        end
+      end
     end
 
-    def subscribe_to_exchange
-      @topic = channel.fanout(config.topic_name)
-      queue.bind(topic, routing_key: queue.name)
+    def subscribe_to_exchange(topic_name, &block)
+      topic_name = topic_name.to_s
+      exchanges[topic_name] ||= begin
+        service.start
+        channel = service.create_channel
+        channel.fanout(topic_name)
+      end
+      block.call(channel, exchanges[topic_name])
     end
 
     def deliver_data(payload)
-      subscribe_to_queue
-      topic.publish(payload.to_json, message_settings)
-
-      # Ugly fix: "IO timeout when reading 7 bytes"
-      # https://stackoverflow.com/questions/39039129/rabbitmq-timeouterror-io-timeout-when-reading-7-bytes
-      channel.close
-      service.close
+      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |_channel, exchange|
+          exchange.publish(encode_payload(payload), message_settings(payload))
+        end
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/subscriber.rb

@@ -5,24 +5,25 @@ module PubSubModelSync
     attr_accessor :klass, :action, :attrs, :settings, :identifiers
     attr_reader :payload
 
-    # @param settings: (Hash) { id: :id, direct_mode: false,
+    # @param settings: (Hash) { id: :id, mode: :model|:klass|:custom_model,
     #                           from_klass: klass, from_action: action }
     def initialize(klass, action, attrs: nil, settings: {})
-      def_settings = { id: :id, direct_mode: false,
-                       from_klass: klass, from_action: action }
+      @settings = { id: settings[:id] || :id,
+                    mode: settings[:mode] || :klass,
+                    from_klass: settings[:from_klass] || klass,
+                    from_action: settings[:from_action] || action }
       @klass = klass
       @action = action
       @attrs = attrs
-      @settings = def_settings.merge(settings)
-      @identifiers = Array(settings[:id]).map(&:to_sym)
+      @identifiers = Array(@settings[:id]).map(&:to_sym)
     end
 
     def process!(payload)
       @payload = payload
-      if settings[:direct_mode]
-        run_class_message
-      else
-        run_model_message
+      case settings[:mode]
+      when :klass then run_class_message
+      when :custom_model then run_model_message(crud_action: false)
+      else run_model_message
       end
     end
 
@@ -34,20 +35,26 @@ module PubSubModelSync
     end
 
     # support for: create, update, destroy
-    def run_model_message
+    def run_model_message(crud_action: true)
       model = find_model
-      return if model.ps_before_save_sync(payload) == :cancel
+      model.ps_processed_payload = payload
+      return model.send(action, payload.data) if ensure_sync(model) && !crud_action
 
       if action == :destroy
-        model.destroy!
+        model.destroy! if ensure_sync(model)
       else
         populate_model(model)
-        return if action == :update && !model.ps_subscriber_changed?(payload.data)
-
-        model.save!
+        model.save! if ensure_sync(model)
       end
     end
 
+    def ensure_sync(model)
+      config = PubSubModelSync::Config
+      cancelled = model.ps_before_save_sync(action, payload) == :cancel
+      config.log("Cancelled sync with ps_before_save_sync: #{[payload]}") if cancelled && config.debug
+      !cancelled
+    end
+
     def find_model
       model_class = klass.constantize
       return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)