pub_sub_model_sync 0.6.0 → 1.0.beta

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class RunSubscriber < Base
+    attr_accessor :subscriber, :payload, :model
+
+    delegate :settings, to: :subscriber
+
+    # @param subscriber(Subscriber)
+    # @param payload(Payload)
+    def initialize(subscriber, payload)
+      @subscriber = subscriber
+      @payload = payload
+    end
+
+    def call
+      klass_subscription? ? run_class_message : run_model_message
+    end
+
+    private
+
+    def klass_subscription?
+      subscriber.settings[:mode] == :klass
+    end
+
+    def run_class_message
+      model_class = subscriber.klass.constantize
+      model_class.ps_processing_payload = payload # TODO: review for parallel notifications
+      call_action(model_class, payload.data) if ensure_sync(model_class)
+    end
+
+    # support for: create, update, destroy
+    def run_model_message
+      @model = find_model
+      model.ps_processing_payload = payload
+      return unless ensure_sync(model)
+
+      populate_model
+      model.send(:ps_before_save_sync) if model.respond_to?(:ps_before_save_sync)
+      call_action(model)
+    end
+
+    def ensure_sync(object)
+      res = true
+      res = false if settings[:if] && !parse_condition(settings[:if], object)
+      res = false if settings[:unless] && parse_condition(settings[:unless], object)
+      log("Cancelled save sync by subscriber condition : #{[payload]}") if !res && debug?
+      res
+    end
+
+    def call_action(object, *args)
+      action_name = settings[:to_action]
+      if action_name.is_a?(Proc)
+        args.prepend(object) unless klass_subscription?
+        action_name.call(*args)
+      else # method name
+        action_name = :save if %i[create update].include?(action_name.to_sym)
+        object.send(action_name, *args)
+      end
+      raise(object.errors) if object.respond_to?(:errors) && object.errors.any?
+    end
+
+    def parse_condition(condition, object)
+      proc_args = klass_subscription? ? [] : [object]
+      case condition
+      when Proc then condition.call(*proc_args)
+      when Array then condition.all? { |method_name| object.send(method_name) }
+      else # method name
+        object.send(condition)
+      end
+    end
+
+    def find_model
+      model_class = subscriber.klass.constantize
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
+
+      model_class.where(model_identifiers).first_or_initialize
+    end
+
+    # @param mappings (Array<String>) supports aliasing, sample: ["id", "full_name:name"]
+    # @return (Hash) hash with the correct attr names and its values
+    def parse_mapping(mappings)
+      mappings.map do |prop|
+        source, target = prop.to_s.split(':')
+        key = (target || source).to_sym
+        next unless payload.data.key?(source.to_sym)
+
+        [key, payload.data[source.to_sym]]
+      end.compact.to_h.symbolize_keys
+    end
+
+    # @return (Hash) hash including identifiers and its values
+    def model_identifiers
+      @model_identifiers ||= parse_mapping(Array(settings[:id]).map(&:to_s))
+    end
+
+    def populate_model
+      values = parse_mapping(subscriber.mapping).except(model_identifiers.keys)
+      values.each do |attr, value|
+        model.send("#{attr}=", value)
+      end
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_base.rb

@@ -43,20 +43,20 @@ module PubSubModelSync
 
     def can_retry_process_message?(error, payload, retries)
       error_payload = [payload, error.message, error.backtrace]
-      if retries == 1
-        log("Error while starting to process message (retrying...): #{error_payload}", :error)
+      if retries <= 5
+        sleep(retries)
+        log("Error while starting to process a message (retrying #{retries} retries...): #{error_payload}", :error)
         rescue_database_connection if lost_db_connection_err?(error)
+        true
       else
-        log("Retried 1 time and error persists, exiting...: #{error_payload}", :error)
+        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
         Process.exit!(true)
       end
-      retries == 1
     end
 
     # @return Payload
     def decode_payload(payload_info)
-      info = JSON.parse(payload_info).deep_symbolize_keys
-      payload = ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+      payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
       log("Received message: #{[payload]}") if config.debug
       payload
     end

data/lib/pub_sub_model_sync/service_google.rb

@@ -33,7 +33,8 @@ module PubSubModelSync
 
     # @param payload (PubSubModelSync::Payload)
     def publish(payload)
-      message_topics = Array(payload.headers[:topic_name] || '').map(&method(:find_topic))
+      p_topic_names = Array(payload.headers[:topic_name] || config.default_topic_name)
+      message_topics = p_topic_names.map(&method(:find_topic))
       message_topics.each do |topic|
         topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
           raise 'Failed to publish the message.' unless res.succeeded?

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -34,7 +34,7 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics = Array(payload.headers[:topic_name] || config.default_topic_name)
       message_topics.each do |topic_name|
         producer.produce(encode_payload(payload), message_settings(payload, topic_name))
       end
@@ -56,8 +56,12 @@ module PubSubModelSync
     end
 
     def start_consumer
-      @consumer = service.consumer(group_id: config.subscription_key)
-      topic_names.each { |topic_name| consumer.subscribe(topic_name) }
+      subscription_key = config.subscription_key
+      @consumer = service.consumer(group_id: subscription_key)
+      topic_names.each do |topic_name|
+        log("Subscribed to topic: #{topic_name} as #{subscription_key}")
+        consumer.subscribe(topic_name)
+      end
     end
 
     def producer

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -73,6 +73,7 @@ module PubSubModelSync
           queue = channel.queue(config.subscription_key, QUEUE_SETTINGS)
           queue.bind(exchange)
           @channels << channel
+          log("Subscribed to topic: #{topic_name} as #{queue.name}")
           block.call(queue)
         end
       end
@@ -89,7 +90,7 @@ module PubSubModelSync
     end
 
     def deliver_data(payload)
-      message_topics = Array(payload.headers[:topic_name] || topic_names.first)
+      message_topics = Array(payload.headers[:topic_name] || config.default_topic_name)
       message_topics.each do |topic_name|
         subscribe_to_exchange(topic_name) do |_channel, exchange|
           exchange.publish(encode_payload(payload), message_settings(payload))

data/lib/pub_sub_model_sync/subscriber.rb

@@ -1,76 +1,22 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Subscriber
-    attr_accessor :klass, :action, :attrs, :settings, :identifiers
-    attr_reader :payload
-
-    # @param settings: (Hash) { id: :id, mode: :model|:klass|:custom_model,
-    #                           from_klass: klass, from_action: action }
-    def initialize(klass, action, attrs: nil, settings: {})
-      @settings = { id: settings[:id] || :id,
-                    mode: settings[:mode] || :klass,
-                    from_klass: settings[:from_klass] || klass,
-                    from_action: settings[:from_action] || action }
+  class Subscriber < PubSubModelSync::Base
+    attr_accessor :klass, :action, :mapping, :settings, :from_klass, :mode
+    attr_reader :payload, :model
+
+    # @param klass (String) class name
+    # @param action (Symbol) @refer SubscriberConcern.ps_subscribe
+    # @param mapping (Array<String>) @refer SubscriberConcern.ps_subscribe
+    # @param settings (Hash): @refer SubscriberConcern.ps_subscribe
+    def initialize(klass, action, mapping: [], settings: {})
+      def_settings = { from_klass: klass, to_action: action, id: :id, if: nil, unless: nil, mode: :model }
       @klass = klass
-      @action = action
-      @attrs = attrs
-      @identifiers = Array(@settings[:id]).map(&:to_sym)
-    end
-
-    def process!(payload)
-      @payload = payload
-      case settings[:mode]
-      when :klass then run_class_message
-      when :custom_model then run_model_message(crud_action: false)
-      else run_model_message
-      end
-    end
-
-    private
-
-    def run_class_message
-      model_class = klass.constantize
-      model_class.send(action, payload.data)
-    end
-
-    # support for: create, update, destroy
-    def run_model_message(crud_action: true)
-      model = find_model
-      model.ps_processed_payload = payload
-      return model.send(action, payload.data) if ensure_sync(model) && !crud_action
-
-      if action == :destroy
-        model.destroy! if ensure_sync(model)
-      else
-        populate_model(model)
-        model.save! if ensure_sync(model)
-      end
-    end
-
-    def ensure_sync(model)
-      config = PubSubModelSync::Config
-      cancelled = model.ps_before_save_sync(action, payload) == :cancel
-      config.log("Cancelled sync with ps_before_save_sync: #{[payload]}") if cancelled && config.debug
-      !cancelled
-    end
-
-    def find_model
-      model_class = klass.constantize
-      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
-
-      model_class.where(model_identifiers).first_or_initialize
-    end
-
-    def model_identifiers
-      identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
-    end
-
-    def populate_model(model)
-      values = payload.data.slice(*attrs).except(*identifiers)
-      values.each do |attr, value|
-        model.send("#{attr}=", value)
-      end
+      @mapping = mapping
+      @settings = def_settings.merge(settings)
+      @action = action.to_sym
+      @from_klass = @settings[:from_klass].to_s
+      @mode = @settings[:mode].to_sym
     end
   end
 end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -4,44 +4,39 @@ module PubSubModelSync
   module SubscriberConcern
     def self.included(base)
       base.extend(ClassMethods)
-      base.send(:attr_accessor, :ps_processed_payload)
+      base.send(:attr_accessor, :ps_processing_payload)
+      base.send(:cattr_accessor, :ps_processing_payload)
     end
 
-    # permit to apply custom actions before applying sync
-    # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
-    def ps_before_save_sync(_action, _payload); end
-
     module ClassMethods
-      def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
-        settings = { id: id, from_klass: from_klass, mode: :model }
-        actions ||= %i[create update destroy]
-        actions.each do |action|
-          add_ps_subscriber(action, attrs, settings)
+      # @param actions (Symbol|Array<Symbol>) Notification.action name: save|create|update|destroy|<any_other_action>
+      # @param mapping (Array<String>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
+      # @param settings (Hash<:from_klass, :to_action, :id, :if, :unless>)
+      #   from_klass (String) Notification.class name
+      #   to_action (Symbol|Proc):
+      #     Symbol: Method to process the notification
+      #     Proc: Block to process the notification
+      #   id (Symbol|Array<Symbol|String>) attribute(s) DB primary identifier(s). Supports for mapping format.
+      #   if (Symbol|Proc|Array<Symbol>) Method or block called as the conformation before calling the callback
+      #   unless (Symbol|Proc|Array<Symbol>) Method or block called as the negation before calling the callback
+      def ps_subscribe(actions, mapping = [], settings = {})
+        Array(actions).each do |action|
+          add_ps_subscriber(action, mapping, settings)
         end
       end
 
-      def ps_subscribe_custom(action, from_klass: name, id: :id, from_action: nil)
-        settings = { id: id, mode: :custom_model, from_klass: from_klass, from_action: from_action }
-        add_ps_subscriber(action, nil, settings)
-      end
-
-      def ps_class_subscribe(action, from_action: nil, from_klass: nil)
-        settings = { mode: :klass, from_action: from_action, from_klass: from_klass }
-        add_ps_subscriber(action, nil, settings)
-      end
-
-      def ps_subscriber(action = :create)
-        PubSubModelSync::Config.subscribers.find do |subscriber|
-          subscriber.klass == name && subscriber.action == action
-        end
+      # @param action (Symbol) Notification.action name
+      # @param settings (Hash) @refer ps_subscribe.settings except(:id)
+      def ps_class_subscribe(action, settings = {})
+        add_ps_subscriber(action, nil, settings.merge(mode: :klass))
       end
 
       private
 
       # @param settings (Hash): refer to PubSubModelSync::Subscriber.settings
-      def add_ps_subscriber(action, attrs, settings = {})
+      def add_ps_subscriber(action, mapping, settings = {})
         klass = PubSubModelSync::Subscriber
-        subscriber = klass.new(name, action, attrs: attrs, settings: settings)
+        subscriber = klass.new(name, action, mapping: mapping, settings: settings)
         PubSubModelSync::Config.subscribers.push(subscriber) && subscriber
       end
     end

data/lib/pub_sub_model_sync/transaction.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Transaction < Base
+    PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
+    attr_accessor :key, :payloads, :use_buffer, :parent, :children
+
+    # @param key (String|nil) Transaction key, if empty will use the ordering_key from first payload
+    # @param use_buffer (Boolean, default: true) If false, payloads are delivered immediately
+    #   (no way to cancel/rollback if transaction failed)
+    def initialize(key, use_buffer: config.transactions_use_buffer)
+      @key = key
+      @use_buffer = use_buffer
+      @children = []
+      @payloads = []
+    end
+
+    # @param payload (Payload)
+    def add_payload(payload)
+      use_buffer ? payloads << payload : deliver_payload(payload)
+    end
+
+    def deliver_all
+      if parent
+        parent.children = parent.children.reject { |t| t == self }
+        parent.deliver_all
+      end
+      payloads.each(&method(:deliver_payload)) if children.empty?
+      clean_publisher
+    end
+
+    def add_transaction(transaction)
+      transaction.parent = self
+      children << transaction
+      transaction
+    end
+
+    def rollback
+      log("rollback #{children.count} notifications", :warn) if children.any? && debug?
+      self.children = []
+      parent&.rollback
+      clean_publisher
+    end
+
+    def clean_publisher
+      PUBLISHER_KLASS.current_transaction = nil if !parent && children.empty?
+    end
+
+    private
+
+    def deliver_payload(payload)
+      PUBLISHER_KLASS.connector_publish(payload)
+    rescue => e
+      PUBLISHER_KLASS.send(:notify_error, e, payload)
+    end
+  end
+end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.6.0'
+  VERSION = '1.0.beta'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 1.0.beta
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-03-04 00:00:00.000000000 Z
+date: 2021-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -119,6 +119,7 @@ files:
 - lib/pub_sub_model_sync/publisher.rb
 - lib/pub_sub_model_sync/publisher_concern.rb
 - lib/pub_sub_model_sync/railtie.rb
+- lib/pub_sub_model_sync/run_subscriber.rb
 - lib/pub_sub_model_sync/runner.rb
 - lib/pub_sub_model_sync/service_base.rb
 - lib/pub_sub_model_sync/service_google.rb
@@ -127,6 +128,7 @@ files:
 - lib/pub_sub_model_sync/subscriber.rb
 - lib/pub_sub_model_sync/subscriber_concern.rb
 - lib/pub_sub_model_sync/tasks/worker.rake
+- lib/pub_sub_model_sync/transaction.rb
 - lib/pub_sub_model_sync/version.rb
 - pub_sub_model_sync.gemspec
 homepage: https://github.com/owen2345/pub_sub_model_sync
@@ -147,9 +149,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.4'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.0.8
 signing_key: