pub_sub_model_sync 0.5.10 → 1.0

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -7,14 +7,20 @@ end
 
 module PubSubModelSync
   class ServiceKafka < ServiceBase
+    QTY_WORKERS = 10
+    LISTEN_SETTINGS = {}.freeze
+    PUBLISH_SETTINGS = {}.freeze
+    PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
     cattr_accessor :producer
-    attr_accessor :config, :service, :consumer
+
+    # @!attribute topic_names (Array): ['topic 1', 'topic 2']
+    attr_accessor :service, :consumer, :topic_names
 
     def initialize
-      @config = PubSubModelSync::Config
       settings = config.kafka_connection
       settings[1][:client_id] ||= config.subscription_key
       @service = Kafka.new(*settings)
+      @topic_names = ensure_topics(Array(config.topic_name || 'model_sync'))
     end
 
     def listen_messages
@@ -28,12 +34,10 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      settings = {
-        topic: config.topic_name,
-        headers: { SERVICE_KEY => true }
-      }.merge(PUBLISH_SETTINGS)
-      producer.produce(payload.to_json, settings)
-      producer.deliver_messages
+      message_topics = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics.each do |topic_name|
+        producer.produce(encode_payload(payload), message_settings(payload, topic_name))
+      end
     end
 
     def stop
@@ -43,22 +47,45 @@ module PubSubModelSync
 
     private
 
+    def message_settings(payload, topic_name)
+      {
+        topic: ensure_topics(topic_name),
+        partition_key: payload.headers[:ordering_key],
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+    end
+
     def start_consumer
-      @consumer = service.consumer(group_id: config.subscription_key)
-      consumer.subscribe(config.topic_name)
+      subscription_key = config.subscription_key
+      @consumer = service.consumer(group_id: subscription_key)
+      topic_names.each do |topic_name|
+        log("Subscribed to topic: #{topic_name} as #{subscription_key}")
+        consumer.subscribe(topic_name)
+      end
     end
 
     def producer
       return self.class.producer if self.class.producer
 
       at_exit { self.class.producer.shutdown }
-      self.class.producer = service.producer
+      self.class.producer = service.async_producer(PRODUCER_SETTINGS)
     end
 
     def process_message(message)
-      return unless message.headers[SERVICE_KEY]
+      super(message.value) if message.headers[SERVICE_KEY]
+    end
 
-      super(message.value)
+    # Check topic existence, create if missing topic
+    # @param names (Array<String>|String)
+    # @return (Array|String) return @param names
+    def ensure_topics(names)
+      missing_topics = Array(names) - (@known_topics || service.topics)
+      missing_topics.each do |name|
+        service.create_topic(name)
+      end
+      @known_topics ||= [] # cache service.topics to reduce verification time
+      @known_topics = (@known_topics + Array(names)).uniq
+      names
     end
   end
 end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -7,18 +7,26 @@ end
 
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
-    attr_accessor :config, :service, :channel, :queue, :topic
+    QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
+    LISTEN_SETTINGS = { manual_ack: false }.freeze
+    PUBLISH_SETTINGS = {}.freeze
+
+    # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
+    # @!attribute channels (Array): [Channel1]
+    # @!attribute exchanges (Hash<key: Exchange>): {topic_name: Exchange1}
+    attr_accessor :service, :topic_names, :channels, :exchanges
 
     def initialize
-      @config = PubSubModelSync::Config
       @service = Bunny.new(*config.bunny_connection)
+      @topic_names = Array(config.topic_name || 'model_sync')
+      @channels = []
+      @exchanges = {}
     end
 
     def listen_messages
       log('Listener starting...')
-      subscribe_to_queue
+      subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
       log('Listener started')
-      queue.subscribe(subscribe_settings, &method(:process_message))
       loop { sleep 5 }
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
@@ -40,54 +48,54 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
-      channel&.close
+      channels.each(&:close)
       service.close
     end
 
     private
 
-    def message_settings
+    def message_settings(payload)
       {
-        routing_key: queue.name,
+        routing_key: payload.headers[:ordering_key],
         type: SERVICE_KEY,
         persistent: true
       }.merge(PUBLISH_SETTINGS)
     end
 
-    def queue_settings
-      { durable: true, auto_delete: false }
-    end
-
-    def subscribe_settings
-      { manual_ack: false }.merge(LISTEN_SETTINGS)
-    end
-
     def process_message(_delivery_info, meta_info, payload)
-      return unless meta_info[:type] == SERVICE_KEY
-
-      super(payload)
+      super(payload) if meta_info[:type] == SERVICE_KEY
     end
 
-    def subscribe_to_queue
-      service.start
-      @channel = service.create_channel
-      @queue = channel.queue(config.subscription_key, queue_settings)
-      subscribe_to_exchange
+    def subscribe_to_queues(&block)
+      @channels = []
+      topic_names.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |channel, exchange|
+          queue = channel.queue(config.subscription_key, QUEUE_SETTINGS)
+          queue.bind(exchange)
+          @channels << channel
+          log("Subscribed to topic: #{topic_name} as #{queue.name}")
+          block.call(queue)
+        end
+      end
     end
 
-    def subscribe_to_exchange
-      @topic = channel.fanout(config.topic_name)
-      queue.bind(topic, routing_key: queue.name)
+    def subscribe_to_exchange(topic_name, &block)
+      topic_name = topic_name.to_s
+      exchanges[topic_name] ||= begin
+        service.start
+        channel = service.create_channel
+        channel.fanout(topic_name)
+      end
+      block.call(channel, exchanges[topic_name])
     end
 
     def deliver_data(payload)
-      subscribe_to_queue
-      topic.publish(payload.to_json, message_settings)
-
-      # Ugly fix: "IO timeout when reading 7 bytes"
-      # https://stackoverflow.com/questions/39039129/rabbitmq-timeouterror-io-timeout-when-reading-7-bytes
-      channel.close
-      service.close
+      message_topics = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics.each do |topic_name|
+        subscribe_to_exchange(topic_name) do |_channel, exchange|
+          exchange.publish(encode_payload(payload), message_settings(payload))
+        end
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/subscriber.rb

@@ -1,74 +1,22 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Subscriber
-    attr_accessor :klass, :action, :attrs, :settings, :identifiers
-    attr_reader :payload
-
-    # @param settings: (Hash) { id: :id, direct_mode: false,
-    #                           from_klass: klass, from_action: action }
-    def initialize(klass, action, attrs: nil, settings: {})
-      def_settings = { id: :id, direct_mode: false,
-                       from_klass: klass, from_action: action }
+  class Subscriber < PubSubModelSync::Base
+    attr_accessor :klass, :action, :mapping, :settings, :from_klass, :mode
+    attr_reader :payload, :model
+
+    # @param klass (String) class name
+    # @param action (Symbol) @refer SubscriberConcern.ps_subscribe
+    # @param mapping (Array<String>) @refer SubscriberConcern.ps_subscribe
+    # @param settings (Hash): @refer SubscriberConcern.ps_subscribe
+    def initialize(klass, action, mapping: [], settings: {})
+      def_settings = { from_klass: klass, to_action: action, id: :id, if: nil, unless: nil, mode: :model }
       @klass = klass
-      @action = action
-      @attrs = attrs
+      @mapping = mapping
       @settings = def_settings.merge(settings)
-      @identifiers = Array(settings[:id]).map(&:to_sym)
-    end
-
-    def process!(payload)
-      @payload = payload
-      if settings[:direct_mode]
-        run_class_message
-      else
-        run_model_message
-      end
-    end
-
-    private
-
-    def run_class_message
-      model_class = klass.constantize
-      model_class.send(action, payload.data)
-    end
-
-    # support for: create, update, destroy
-    def run_model_message
-      model = find_model
-      model.ps_processed_payload = payload
-
-      if action == :destroy
-        model.destroy! if ensure_sync(model)
-      else
-        populate_model(model)
-        model.save! if ensure_sync(model)
-      end
-    end
-
-    def ensure_sync(model)
-      config = PubSubModelSync::Config
-      cancelled = model.ps_before_save_sync(payload) == :cancel
-      config.log("Cancelled sync with ps_before_save_sync: #{[payload]}") if cancelled && config.debug
-      !cancelled
-    end
-
-    def find_model
-      model_class = klass.constantize
-      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
-
-      model_class.where(model_identifiers).first_or_initialize
-    end
-
-    def model_identifiers
-      identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
-    end
-
-    def populate_model(model)
-      values = payload.data.slice(*attrs).except(*identifiers)
-      values.each do |attr, value|
-        model.send("#{attr}=", value)
-      end
+      @action = action.to_sym
+      @from_klass = @settings[:from_klass].to_s
+      @mode = @settings[:mode].to_sym
     end
   end
 end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -4,41 +4,41 @@ module PubSubModelSync
   module SubscriberConcern
     def self.included(base)
       base.extend(ClassMethods)
-      base.send(:attr_accessor, :ps_processed_payload)
+      base.send(:attr_accessor, :ps_processing_payload)
+      base.send(:cattr_accessor, :ps_processing_payload)
     end
 
-    # permit to apply custom actions before applying sync
-    # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
-    def ps_before_save_sync(_payload); end
-
     module ClassMethods
-      def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
-        settings = { id: id, from_klass: from_klass }
-        actions ||= %i[create update destroy]
-        actions.each do |action|
-          add_ps_subscriber(action, attrs, settings)
+      # @param actions (Symbol|Array<Symbol>) Notification.action name: save|create|update|destroy|<any_other_action>
+      # @param mapping (Array<String,Symbol>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
+      # @param settings (Hash<:from_klass, :to_action, :id, :if, :unless>)
+      #   from_klass (String) Notification.class name
+      #   to_action (Symbol|Proc):
+      #     Symbol: Method to process the notification
+      #     Proc: Block to process the notification
+      #   id (Symbol|Array<Symbol|String>) attribute(s) DB primary identifier(s). Supports for mapping format.
+      #   if (Symbol|Proc|Array<Symbol>) Method or block called as the conformation before calling the callback
+      #   unless (Symbol|Proc|Array<Symbol>) Method or block called as the negation before calling the callback
+      def ps_subscribe(actions, mapping = [], settings = {}, &block)
+        settings[:to_action] ||= block if block
+        Array(actions).map do |action|
+          add_ps_subscriber(action, mapping, settings)
         end
       end
 
-      def ps_class_subscribe(action, from_action: nil, from_klass: nil)
-        settings = { direct_mode: true }
-        settings[:from_action] = from_action if from_action
-        settings[:from_klass] = from_klass if from_klass
-        add_ps_subscriber(action, nil, settings)
-      end
-
-      def ps_subscriber(action = :create)
-        PubSubModelSync::Config.subscribers.find do |subscriber|
-          subscriber.klass == name && subscriber.action == action
-        end
+      # @param action (Symbol) Notification.action name
+      # @param settings (Hash) @refer ps_subscribe.settings except(:id)
+      def ps_class_subscribe(action, settings = {}, &block)
+        settings[:to_action] ||= block if block
+        add_ps_subscriber(action, nil, settings.merge(mode: :klass))
       end
 
       private
 
       # @param settings (Hash): refer to PubSubModelSync::Subscriber.settings
-      def add_ps_subscriber(action, attrs, settings = {})
+      def add_ps_subscriber(action, mapping, settings = {})
         klass = PubSubModelSync::Subscriber
-        subscriber = klass.new(name, action, attrs: attrs, settings: settings)
+        subscriber = klass.new(name, action, mapping: mapping, settings: settings)
         PubSubModelSync::Config.subscribers.push(subscriber) && subscriber
       end
     end

data/lib/pub_sub_model_sync/tasks/worker.rake

@@ -3,6 +3,17 @@
 namespace :pub_sub_model_sync do
   desc 'Start listening syncs'
   task start: :environment do
+    # https://github.com/zendesk/ruby-kafka#consumer-groups
+    # Each consumer process will be assigned one or more partitions from each topic that the group
+    #   subscribes to. In order to handle more messages, simply start more processes.
+    if PubSubModelSync::Config.service_name == :kafka
+      (PubSubModelSync::ServiceKafka::QTY_WORKERS - 1).times.each do
+        Thread.new do
+          Thread.current.abort_on_exception = true
+          PubSubModelSync::Runner.new.run
+        end
+      end
+    end
     PubSubModelSync::Runner.new.run
   end
 end

data/lib/pub_sub_model_sync/transaction.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Transaction < Base
+    PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
+    attr_accessor :key, :payloads, :max_buffer, :root, :children, :finished
+
+    # @param key (String|nil) Transaction key, if empty will use the ordering_key from first payload
+    # @param max_buffer (Integer) Once this quantity of notifications is reached, then all notifications
+    #   will immediately be delivered.
+    #   Note: There is no way to rollback delivered notifications if current transaction fails
+    def initialize(key, max_buffer: config.transactions_max_buffer)
+      @key = key
+      @max_buffer = max_buffer
+      @children = []
+      @payloads = []
+    end
+
+    # @param payload (Payload)
+    def add_payload(payload)
+      payloads << payload
+      log("Payload added to current transaction: #{payload.inspect}") if config.debug
+      return unless payloads.count >= max_buffer
+
+      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
+      deliver_payloads
+    end
+
+    def finish # rubocop:disable Metrics/AbcSize
+      if root
+        root.children = root.children.reject { |t| t == self }
+        root.deliver_all if root.finished && root.children.empty?
+      end
+      self.finished = true
+      deliver_all if children.empty?
+    end
+
+    def add_transaction(transaction)
+      transaction.root = self
+      children << transaction
+      transaction
+    end
+
+    def rollback
+      log("Rollback #{payloads.count} notifications", :warn) if children.any? && debug?
+      self.children = []
+      root&.rollback
+      clean_publisher
+    end
+
+    def clean_publisher
+      PUBLISHER_KLASS.current_transaction = nil if !root && children.empty?
+    end
+
+    def deliver_all
+      deliver_payloads
+      clean_publisher
+    end
+
+    private
+
+    def deliver_payloads
+      payloads.each do |payload|
+        begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
+          PUBLISHER_KLASS.connector_publish(payload)
+        rescue => e
+          PUBLISHER_KLASS.send(:notify_error, e, payload)
+        end
+      end
+      self.payloads = []
+    end
+  end
+end