pub_sub_model_sync 0.5.10 → 0.6.0

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,8 +4,6 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
-    PUBLISH_SETTINGS = {}.freeze
-    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'
@@ -22,10 +20,20 @@ module PubSubModelSync
 
     private
 
+    # @param payload (Payload)
+    # @return (String): Json Format
+    def encode_payload(payload)
+      data = payload.to_h
+      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      reduce_payload_size = !config.debug
+      data[:headers].except!(*not_important_keys) if reduce_payload_size
+      data.to_json
+    end
+
     # @param (String: Payload in json format)
     def process_message(payload_info)
       retries ||= 0
-      payload = parse_payload(payload_info)
+      payload = decode_payload(payload_info)
       return payload.process unless same_app_message?(payload)
 
       log("Skipping message from same origin: #{[payload]}") if config.debug
@@ -45,7 +53,8 @@ module PubSubModelSync
       retries == 1
     end
 
-    def parse_payload(payload_info)
+    # @return Payload
+    def decode_payload(payload_info)
       info = JSON.parse(payload_info).deep_symbolize_keys
       payload = ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
       log("Received message: #{[payload]}") if config.debug

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,50 +7,85 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true }.freeze
+    LISTEN_SETTINGS = { message_ordering: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
     TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
-    attr_accessor :service, :topic, :subscription, :subscriber
+
+    # @!attribute topics (Hash): { key: Topic1, ... }
+    # @!attribute publish_topics (Hash): { key: Topic1, ... }
+    attr_accessor :service, :topics, :subscribers, :publish_topics
 
     def initialize
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
-      @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name, TOPIC_SETTINGS)
-      topic.enable_message_ordering!
+      Array(config.topic_name || 'model_sync').each(&method(:init_topic))
     end
 
     def listen_messages
-      @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
-      subscriber.start
+      @subscribers = subscribe_to_topics
       log('Listener started')
       sleep
-      subscriber.stop.wait!
+      subscribers.each { |subscriber| subscriber.stop.wait! }
       log('Listener stopped')
     end
 
+    # @param payload (PubSubModelSync::Payload)
     def publish(payload)
-      topic.publish_async(payload.to_json, message_headers) do |res|
-        raise 'Failed to publish the message.' unless res.succeeded?
+      message_topics = Array(payload.headers[:topic_name] || '').map(&method(:find_topic))
+      message_topics.each do |topic|
+        topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
+          raise 'Failed to publish the message.' unless res.succeeded?
+        end
       end
     end
 
     def stop
       log('Listener stopping...')
-      subscriber.stop!
+      subscribers.each(&:stop!)
     end
 
     private
 
-    def message_headers
-      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    def find_topic(topic_name)
+      topic_name = topic_name.to_s
+      return topics.values.first unless topic_name.present?
+
+      topics[topic_name] || publish_topics[topic_name] || init_topic(topic_name, only_publish: true)
     end
 
-    def subscribe_to_topic
-      topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
+    # @param only_publish (Boolean): if false is used to listen and publish messages
+    # @return (Topic): returns created or loaded topic
+    def init_topic(topic_name, only_publish: false)
+      topic_name = topic_name.to_s
+      @topics ||= {}
+      @publish_topics ||= {}
+      topic = service.topic(topic_name) || service.create_topic(topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
+      publish_topics[topic_name] = topic if only_publish
+      topics[topic_name] = topic unless only_publish
+      topic
+    end
+
+    # @param payload (PubSubModelSync::Payload)
+    def message_headers(payload)
+      {
+        SERVICE_KEY => true,
+        ordering_key: payload.headers[:ordering_key]
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    # @return [Subscriber]
+    def subscribe_to_topics
+      topics.map do |key, topic|
+        subs_name = "#{config.subscription_key}_#{key}"
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscriber.start
+        log("Subscribed to topic: #{topic.name} as: #{subs_name}")
+        subscriber
+      end
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -7,14 +7,20 @@ end
 
 module PubSubModelSync
   class ServiceKafka < ServiceBase
+    QTY_WORKERS = 10
+    LISTEN_SETTINGS = {}.freeze
+    PUBLISH_SETTINGS = {}.freeze
+    PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
     cattr_accessor :producer
-    attr_accessor :config, :service, :consumer
+
+    # @!attribute topic_names (Array): ['topic 1', 'topic 2']
+    attr_accessor :service, :consumer, :topic_names
 
     def initialize
-      @config = PubSubModelSync::Config
       settings = config.kafka_connection
       settings[1][:client_id] ||= config.subscription_key
       @service = Kafka.new(*settings)
+      @topic_names = ensure_topics(Array(config.topic_name || 'model_sync'))
     end
 
     def listen_messages
@@ -28,12 +34,10 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      settings = {
-        topic: config.topic_name,
-        headers: { SERVICE_KEY => true }
-      }.merge(PUBLISH_SETTINGS)
-      producer.produce(payload.to_json, settings)
-      producer.deliver_messages
+      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics.each do |topic_name|
+        producer.produce(encode_payload(payload), message_settings(payload, topic_name))
+      end
     end
 
     def stop
@@ -43,22 +47,41 @@ module PubSubModelSync
 
     private
 
+    def message_settings(payload, topic_name)
+      {
+        topic: ensure_topics(topic_name),
+        partition_key: payload.headers[:ordering_key],
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+    end
+
     def start_consumer
       @consumer = service.consumer(group_id: config.subscription_key)
-      consumer.subscribe(config.topic_name)
+      topic_names.each { |topic_name| consumer.subscribe(topic_name) }
     end
 
     def producer
       return self.class.producer if self.class.producer
 
       at_exit { self.class.producer.shutdown }
-      self.class.producer = service.producer
+      self.class.producer = service.async_producer(PRODUCER_SETTINGS)
     end
 
     def process_message(message)
-      return unless message.headers[SERVICE_KEY]
+      super(message.value) if message.headers[SERVICE_KEY]
+    end
 
-      super(message.value)
+    # Check topic existence, create if missing topic
+    # @param names (Array<String>|String)
+    # @return (Array|String) return @param names
+    def ensure_topics(names)
+      missing_topics = Array(names) - (@known_topics || service.topics)
+      missing_topics.each do |name|
+        service.create_topic(name)
+      end
+      @known_topics ||= [] # cache service.topics to reduce verification time
+      @known_topics = (@known_topics + Array(names)).uniq
+      names
     end
   end
 end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -7,18 +7,26 @@ end
 
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
-    attr_accessor :config, :service, :channel, :queue, :topic
+    QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
+    LISTEN_SETTINGS = { manual_ack: true }.freeze
+    PUBLISH_SETTINGS = {}.freeze
+
+    # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
+    # @!attribute channels (Array): [Channel1]
+    # @!attribute exchanges (Hash<key: Exchange>): {topic_name: Exchange1}
+    attr_accessor :service, :topic_names, :channels, :exchanges
 
     def initialize
-      @config = PubSubModelSync::Config
       @service = Bunny.new(*config.bunny_connection)
+      @topic_names = Array(config.topic_name || 'model_sync')
+      @channels = []
+      @exchanges = {}
     end
 
     def listen_messages
       log('Listener starting...')
-      subscribe_to_queue
+      subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
       log('Listener started')
-      queue.subscribe(subscribe_settings, &method(:process_message))
       loop { sleep 5 }
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
@@ -40,54 +48,53 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
-      channel&.close
+      channels.each(&:close)
       service.close
     end
 
     private
 
-    def message_settings
+    def message_settings(payload)
       {
-        routing_key: queue.name,
+        routing_key: payload.headers[:ordering_key],
         type: SERVICE_KEY,
         persistent: true
       }.merge(PUBLISH_SETTINGS)
     end
 
-    def queue_settings
-      { durable: true, auto_delete: false }
-    end
-
-    def subscribe_settings
-      { manual_ack: false }.merge(LISTEN_SETTINGS)
-    end
-
     def process_message(_delivery_info, meta_info, payload)
-      return unless meta_info[:type] == SERVICE_KEY
-
-      super(payload)
+      super(payload) if meta_info[:type] == SERVICE_KEY
     end
 
-    def subscribe_to_queue
-      service.start
-      @channel = service.create_channel
-      @queue = channel.queue(config.subscription_key, queue_settings)
-      subscribe_to_exchange
+    def subscribe_to_queues(&block)
+      @channels = []
+      topic_names.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |channel, exchange|
+          queue = channel.queue(config.subscription_key, QUEUE_SETTINGS)
+          queue.bind(exchange)
+          @channels << channel
+          block.call(queue)
+        end
+      end
     end
 
-    def subscribe_to_exchange
-      @topic = channel.fanout(config.topic_name)
-      queue.bind(topic, routing_key: queue.name)
+    def subscribe_to_exchange(topic_name, &block)
+      topic_name = topic_name.to_s
+      exchanges[topic_name] ||= begin
+        service.start
+        channel = service.create_channel
+        channel.fanout(topic_name)
+      end
+      block.call(channel, exchanges[topic_name])
     end
 
     def deliver_data(payload)
-      subscribe_to_queue
-      topic.publish(payload.to_json, message_settings)
-
-      # Ugly fix: "IO timeout when reading 7 bytes"
-      # https://stackoverflow.com/questions/39039129/rabbitmq-timeouterror-io-timeout-when-reading-7-bytes
-      channel.close
-      service.close
+      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |_channel, exchange|
+          exchange.publish(encode_payload(payload), message_settings(payload))
+        end
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/subscriber.rb

@@ -5,24 +5,25 @@ module PubSubModelSync
     attr_accessor :klass, :action, :attrs, :settings, :identifiers
     attr_reader :payload
 
-    # @param settings: (Hash) { id: :id, direct_mode: false,
+    # @param settings: (Hash) { id: :id, mode: :model|:klass|:custom_model,
     #                           from_klass: klass, from_action: action }
     def initialize(klass, action, attrs: nil, settings: {})
-      def_settings = { id: :id, direct_mode: false,
-                       from_klass: klass, from_action: action }
+      @settings = { id: settings[:id] || :id,
+                    mode: settings[:mode] || :klass,
+                    from_klass: settings[:from_klass] || klass,
+                    from_action: settings[:from_action] || action }
       @klass = klass
       @action = action
       @attrs = attrs
-      @settings = def_settings.merge(settings)
-      @identifiers = Array(settings[:id]).map(&:to_sym)
+      @identifiers = Array(@settings[:id]).map(&:to_sym)
     end
 
     def process!(payload)
       @payload = payload
-      if settings[:direct_mode]
-        run_class_message
-      else
-        run_model_message
+      case settings[:mode]
+      when :klass then run_class_message
+      when :custom_model then run_model_message(crud_action: false)
+      else run_model_message
       end
     end
 
@@ -34,9 +35,10 @@ module PubSubModelSync
     end
 
     # support for: create, update, destroy
-    def run_model_message
+    def run_model_message(crud_action: true)
       model = find_model
       model.ps_processed_payload = payload
+      return model.send(action, payload.data) if ensure_sync(model) && !crud_action
 
       if action == :destroy
         model.destroy! if ensure_sync(model)
@@ -48,7 +50,7 @@ module PubSubModelSync
 
     def ensure_sync(model)
       config = PubSubModelSync::Config
-      cancelled = model.ps_before_save_sync(payload) == :cancel
+      cancelled = model.ps_before_save_sync(action, payload) == :cancel
       config.log("Cancelled sync with ps_before_save_sync: #{[payload]}") if cancelled && config.debug
       !cancelled
     end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -9,21 +9,24 @@ module PubSubModelSync
 
     # permit to apply custom actions before applying sync
     # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
-    def ps_before_save_sync(_payload); end
+    def ps_before_save_sync(_action, _payload); end
 
     module ClassMethods
       def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
-        settings = { id: id, from_klass: from_klass }
+        settings = { id: id, from_klass: from_klass, mode: :model }
         actions ||= %i[create update destroy]
         actions.each do |action|
           add_ps_subscriber(action, attrs, settings)
         end
       end
 
+      def ps_subscribe_custom(action, from_klass: name, id: :id, from_action: nil)
+        settings = { id: id, mode: :custom_model, from_klass: from_klass, from_action: from_action }
+        add_ps_subscriber(action, nil, settings)
+      end
+
       def ps_class_subscribe(action, from_action: nil, from_klass: nil)
-        settings = { direct_mode: true }
-        settings[:from_action] = from_action if from_action
-        settings[:from_klass] = from_klass if from_klass
+        settings = { mode: :klass, from_action: from_action, from_klass: from_klass }
         add_ps_subscriber(action, nil, settings)
       end
 

data/lib/pub_sub_model_sync/tasks/worker.rake

@@ -3,6 +3,17 @@
 namespace :pub_sub_model_sync do
   desc 'Start listening syncs'
   task start: :environment do
+    # https://github.com/zendesk/ruby-kafka#consumer-groups
+    # Each consumer process will be assigned one or more partitions from each topic that the group
+    #   subscribes to. In order to handle more messages, simply start more processes.
+    if PubSubModelSync::Config.service_name == :kafka
+      (PubSubModelSync::ServiceKafka::QTY_WORKERS - 1).times.each do
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          PubSubModelSync::Runner.new.run
+        end
+      end
+    end
     PubSubModelSync::Runner.new.run
   end
 end