pub_sub_model_sync 0.6.0 → 1.0.beta

data/docker-compose.yaml

@@ -5,7 +5,7 @@ services:
     build: .
     command: sh -c 'bundle exec rspec'
     volumes:
-      - .:/myapp
+      - .:/app
       - bundler_gems:/usr/local/bundle/
 
 volumes:

data/lib/pub_sub_model_sync.rb

@@ -10,8 +10,10 @@ require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'
 require 'pub_sub_model_sync/runner'
+require 'pub_sub_model_sync/transaction'
 require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
+require 'pub_sub_model_sync/run_subscriber'
 
 require 'pub_sub_model_sync/publisher'
 require 'pub_sub_model_sync/subscriber'

data/lib/pub_sub_model_sync/base.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Base
-    delegate :config, :log, to: self
+    delegate :config, :log, :debug?, to: self
 
     class << self
       def config
@@ -12,6 +12,10 @@ module PubSubModelSync
       def log(message, kind = :info)
         config.log message, kind
       end
+
+      def debug?
+        config.debug
+      end
     end
 
     # @param errors (Array(Class|String))

data/lib/pub_sub_model_sync/config.rb

@@ -3,12 +3,12 @@
 module PubSubModelSync
   class Config
     cattr_accessor(:subscribers) { [] }
-    cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
 
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
+    cattr_accessor(:transactions_use_buffer) { true }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
@@ -16,16 +16,15 @@ module PubSubModelSync
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
     cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
-    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
-    cattr_accessor :project, :credentials, :topic_name, :subscription_name
+    cattr_accessor :project, :credentials, :topic_name, :subscription_name, :default_topic_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :topic_name, :subscription_name
+    cattr_accessor :bunny_connection, :topic_name, :subscription_name, :default_topic_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name, :subscription_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name, :default_topic_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
@@ -37,8 +36,17 @@ module PubSubModelSync
     end
 
     def self.subscription_key
-      subscription_name ||
-        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+      klass = Rails.application.class
+      app_name = klass.respond_to?(:module_parent_name) ? klass.module_parent_name : klass.parent_name
+      subscription_name || app_name
+    end
+
+    class << self
+      alias default_topic_name_old default_topic_name
+
+      def default_topic_name
+        default_topic_name_old || Array(topic_name).first
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/message_processor.rb

@@ -28,12 +28,12 @@ module PubSubModelSync
     private
 
     def run_subscriber(subscriber)
-      subscriber = subscriber.dup
+      processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
       errors = [ActiveRecord::ConnectionTimeoutError, 'deadlock detected', 'could not serialize access']
-      retry_error(errors, qty: 2) do
-        subscriber.process!(payload)
+      retry_error(errors, qty: 5) do
+        processor.call
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log
       end
@@ -54,8 +54,7 @@ module PubSubModelSync
 
     def filter_subscribers
       config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
-          subscriber.settings[:from_action].to_s == payload.action.to_s
+        subscriber.from_klass == payload.klass && subscriber.action == payload.action && payload.mode == subscriber.mode
       end
     end
   end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -4,7 +4,7 @@ module PubSubModelSync
   class MessagePublisher < PubSubModelSync::Base
     class << self
       class MissingPublisher < StandardError; end
-      attr_accessor :transaction_key
+      attr_accessor :current_transaction
 
       def connector
         @connector ||= PubSubModelSync::Connector.new
@@ -16,70 +16,53 @@ module PubSubModelSync
       #   be processed in the correct order, despite of the multiple threads. This thanks to the fact
       #   that Pub/Sub services will always send messages with the same `ordering_key` into the same
       #   worker/thread.
-      # @param key (String): This key will be used as the ordering_key for all payload
-      #     inside this transaction.
-      def transaction(key, &block)
-        parent_key = init_transaction(key)
-        begin
-          block.call
-        ensure
-          end_transaction(parent_key)
-        end
+      # @see Transaction.new(...)
+      # @param key (String|Nil)
+      # @param block (Yield) block to be executed
+      def transaction(key, settings = {}, &block)
+        t = init_transaction(key, settings)
+        block.call
+        t.deliver_all
+      rescue
+        t.rollback
+        raise
+      ensure
+        t.clean_publisher
       end
 
       # Starts a new transaction
-      # @return (String) returns parent transaction key
-      def init_transaction(key)
-        parent_key = transaction_key
-        self.transaction_key = transaction_key.presence || key
-        parent_key
-      end
-
-      # Restores to the last transaction key
-      def end_transaction(parent_key)
-        self.transaction_key = parent_key
+      # @param key (@transaction_key)
+      # @return (Transaction)
+      def init_transaction(key, settings = {})
+        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        if current_transaction
+          current_transaction.add_transaction(new_transaction)
+        else
+          self.current_transaction = new_transaction
+        end
+        new_transaction
       end
 
-      # Publishes any value to pubsub
-      # @param klass (String): Class name
-      # @param data (Hash): Data to be delivered
-      # @param action (:symbol): action name
-      # @param headers (Hash, optional): header settings (More in Payload.headers)
+      # Publishes a class level notification via pubsub
+      # @refer PublisherConcern.ps_class_publish
       # @return Payload
       def publish_data(klass, data, action, headers: {})
-        attrs = { klass: klass.to_s, action: action.to_sym }
+        attrs = { klass: klass.to_s, action: action.to_sym, mode: :klass }
         payload = PubSubModelSync::Payload.new(data, attrs, headers)
         publish(payload)
       end
 
-      # Publishes custom model action
-      # @param model (ActiveRecord): Model object owner of the data
-      # @param data (Hash): Data to be delivered
-      # @param action (:symbol): action name
-      # @param as_klass (String, optional): Class name (default model class name)
-      # @param headers (Hash, optional): header settings (More in Payload.headers)
-      # @return Payload
-      def publish_model_data(model, data, action, as_klass: nil, headers: {})
-        headers = PubSubModelSync::Publisher.headers_for(model, action).merge(headers)
-        publish_data(as_klass || model.class.name, data, action, headers: headers)
-      end
-
-      # Publishes model info to pubsub
-      # @param model (ActiveRecord model)
-      # @param action (Sym): Action name
-      # @param custom_data (Hash, optional): If present custom_data will be used as the payload data.
-      # @param custom_headers (Hash): Refer Payload.headers
-      # @return Payload
-      def publish_model(model, action, custom_data: nil, custom_headers: {})
-        return if model.ps_skip_sync?(action)
+      # @param model (ActiveRecord::Base)
+      # @param action (Symbol: @see PublishConcern::ps_publish)
+      # @param settings (Hash: @see Publisher.new.settings)
+      def publish_model(model, action, settings = {})
+        return if model.ps_skip_publish?(action)
 
-        publisher = model.class.ps_publisher(action)
-        error_msg = "No publisher found for: \"#{[model.class.name, action]}\""
-        raise(MissingPublisher, error_msg) unless publisher
+        publisher = PubSubModelSync::Publisher.new(model, action, settings)
+        payload = publisher.payload
 
-        payload = publisher.payload(model, action, custom_data: custom_data, custom_headers: custom_headers)
-        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_sync syncs
-          publish(payload) { model.ps_after_sync(action, payload) } if ensure_model_publish(model, action, payload)
+        transaction(payload.headers[:ordering_key]) do # catch and group all :ps_before_publish syncs
+          publish(payload) { model.ps_after_publish(action, payload) } if ensure_model_publish(model, action, payload)
         end
       end
 
@@ -88,15 +71,20 @@ module PubSubModelSync
       # @return Payload
       # Raises error if exist
       def publish!(payload, &block)
+        payload.headers[:ordering_key] = ordering_key_for(payload)
         return unless ensure_publish(payload)
 
-        log("Publishing message: #{[payload]}")
-        connector.publish(payload)
-        config.on_after_publish.call(payload)
+        current_transaction ? current_transaction.add_payload(payload) : connector_publish(payload)
         block&.call
         payload
       end
 
+      def connector_publish(payload)
+        connector.publish(payload)
+        log("Published message: #{[payload]}")
+        config.on_after_publish.call(payload)
+      end
+
       # Similar to :publish! method
       # Notifies error via :on_error_publish instead of raising error
       # @return Payload
@@ -109,18 +97,20 @@ module PubSubModelSync
       private
 
       def ensure_publish(payload)
-        payload.headers[:ordering_key] = @transaction_key if @transaction_key.present?
-        forced_ordering_key = payload.headers[:forced_ordering_key]
-        payload.headers[:ordering_key] = forced_ordering_key if forced_ordering_key
         cancelled = config.on_before_publish.call(payload) == :cancel
         log("Publish cancelled by config.on_before_publish: #{payload}") if config.debug && cancelled
         !cancelled
       end
 
+      def ordering_key_for(payload)
+        current_transaction&.key ||= payload.headers[:ordering_key]
+        payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]
+      end
+
       def ensure_model_publish(model, action, payload)
-        res_before = model.ps_before_sync(action, payload)
+        res_before = model.ps_before_publish(action, payload)
         cancelled = res_before == :cancel
-        log("Publish cancelled by model.ps_before_sync: #{payload}") if config.debug && cancelled
+        log("Publish cancelled by model.ps_before_publish: #{payload}") if config.debug && cancelled
         !cancelled
       end
 

data/lib/pub_sub_model_sync/payload.rb

@@ -3,10 +3,10 @@
 module PubSubModelSync
   class Payload
     class MissingInfo < StandardError; end
-    attr_reader :data, :attributes, :headers
+    attr_reader :data, :info, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass*: string, action*: :sym }):
+    # @param info (Hash: { klass*: string, action*: :sym, mode?: :klass|:model }):
     # @param headers (Hash):
     #   key (String): identifier of the payload, default:
     #        klass/action: when class message
@@ -19,9 +19,9 @@ module PubSubModelSync
     #     message (default first topic)
     #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
     #     even withing transactions. Default nil.
-    def initialize(data, attributes, headers = {})
+    def initialize(data, info, headers = {})
       @data = data
-      @attributes = attributes
+      @info = { mode: :model }.merge(info)
       @headers = headers
       build_headers
       validate!
@@ -29,15 +29,19 @@ module PubSubModelSync
 
     # @return Hash: payload data
     def to_h
-      { data: data, attributes: attributes, headers: headers }
+      { data: data, info: info, headers: headers }
     end
 
     def klass
-      attributes[:klass].to_s
+      info[:klass].to_s
     end
 
     def action
-      attributes[:action]
+      info[:action].to_sym
+    end
+
+    def mode
+      info[:mode].to_sym
     end
 
     # Process payload data
@@ -69,10 +73,10 @@ module PubSubModelSync
     end
 
     # convert payload data into Payload
-    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)
       data = data.deep_symbolize_keys
-      new(data[:data], data[:attributes], data[:headers])
+      new(data[:data], data[:info], data[:headers])
     end
 
     private
@@ -85,7 +89,7 @@ module PubSubModelSync
     end
 
     def validate!
-      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+      raise MissingInfo if !info[:klass] || !info[:action]
     end
   end
 end

data/lib/pub_sub_model_sync/publisher.rb

@@ -1,31 +1,26 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class Publisher
-    attr_accessor :attrs, :actions, :klass, :as_klass, :headers
-
-    # @param headers (Hash): refer Payload.headers
-    def initialize(attrs, klass, actions = nil, as_klass: nil, headers: {})
-      @attrs = attrs
-      @klass = klass
-      @actions = actions || %i[create update destroy]
-      @as_klass = as_klass || klass
-      @headers = headers
+  class Publisher < PubSubModelSync::Base
+    attr_accessor :model, :action, :data, :mapping, :headers, :as_klass
+
+    # @param model (ActiveRecord::Base)
+    # @param action (@see PublishConcern::ps_publish)
+    # @param settings (@see PublishConcern::ps_publish): { data:, mapping:, headers:, as_klass: }
+    def initialize(model, action, settings = {})
+      @model = model
+      @action = action
+      @data = settings[:data] || {}
+      @mapping = settings[:mapping] || []
+      @headers = settings[:headers] || {}
+      @as_klass = settings[:as_klass] || model.class.name
     end
 
-    # Builds the payload with model information defined for :action (:create|:update|:destroy)
-    # @param custom_headers (Hash, default {}): refer Payload.headers
-    def payload(model, action, custom_data: nil, custom_headers: {})
-      payload_headers = self.class.headers_for(model, action)
-                            .merge(headers)
-                            .merge(custom_headers)
-      data = custom_data || payload_data(model)
-      PubSubModelSync::Payload.new(data, payload_attrs(model, action), payload_headers)
-    end
-
-    def self.headers_for(model, action)
-      key = [model.class.name, action, model.id].join('/')
-      { ordering_key: ordering_key_for(model), key: key }
+    # @return (Payload)
+    def payload
+      values = compute_value(data)
+      values = mapping_data.merge(values)
+      PubSubModelSync::Payload.new(values, settings_data, headers_data)
     end
 
     def self.ordering_key_for(model)
@@ -34,18 +29,29 @@ module PubSubModelSync
 
     private
 
-    def payload_data(model)
-      @attrs.map do |prop|
+    def headers_data
+      klass_name = model.class.name
+      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      def_data.merge(compute_value(headers))
+    end
+
+    def compute_value(value)
+      res = value
+      res = model.send(value, action) if value.is_a?(Symbol) # method name
+      res = value.call(model, action) if value.is_a?(Proc)
+      res
+    end
+
+    def settings_data
+      { klass: as_klass, action: action }
+    end
+
+    def mapping_data
+      mapping.map do |prop|
         source, target = prop.to_s.split(':')
         [target || source, model.send(source.to_sym)]
       end.to_h.symbolize_keys
     end
-
-    def payload_attrs(model, action)
-      {
-        klass: (as_klass || model.class.name).to_s,
-        action: action.to_sym
-      }
-    end
   end
 end

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -2,90 +2,83 @@
 
 module PubSubModelSync
   module PublisherConcern
-    def self.included(base)
-      base.extend(ClassMethods)
-      base.send(:ps_init_transaction_callbacks)
-    end
+    extend ActiveSupport::Concern
 
-    # Before initializing sync service (callbacks: after create/update/destroy)
-    def ps_skip_callback?(_action)
-      false
+    included do
+      extend ClassMethods
+      ps_init_transaction_callbacks if self <= ActiveRecord::Base
     end
 
     # before preparing data to sync
-    def ps_skip_sync?(_action)
+    def ps_skip_publish?(_action)
       false
     end
+    alias ps_skip_sync? ps_skip_publish? # @deprecated
 
     # before delivering data (return :cancel to cancel sync)
-    def ps_before_sync(_action, _payload); end
+    def ps_before_publish(_action, _payload); end
+    alias ps_before_sync ps_before_publish # @deprecated
 
     # after delivering data
-    def ps_after_sync(_action, _payload); end
+    def ps_after_publish(_action, _payload); end
+    alias ps_after_sync ps_after_publish # @deprecated
 
-    # To perform sync on demand
-    # @param action (Sym): CRUD action name
-    # @param custom_data (nil|Hash) If present custom_data will be used as the payload data. I.E.
-    #   data generator will be ignored
-    # @param custom_headers (Hash, optional): refer Payload.headers
-    def ps_perform_sync(action = :create, custom_data: nil, custom_headers: {})
+    # Delivers a notification via pubsub
+    # @param action (Sym|String) Sample: create|update|save|destroy|<any_other_key>
+    # @param mapping? (Array<String>) If present will generate data using the mapping and added to the payload.
+    #   Sample: ["id", "full_name:name"]
+    # @param data? (Hash|Symbol|Proc)
+    #   Hash: Data to be added to the payload
+    #   Symbol: Method name to be called to retrieve payload data (must return a hash value, receives :action name)
+    #   Proc: Block to be called to retrieve payload data
+    # @param headers? (Hash|Symbol|Proc): (All available attributes in Payload.headers)
+    #   Hash: Data that will be merged with default header values
+    #   Symbol: Method name that will be called to retrieve header values (must return a hash, receives :action name)
+    #   Proc: Block to be called to retrieve header values
+    # @param as_klass? (String): Output class name used instead of current class name
+    def ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: self.class.name)
       p_klass = PubSubModelSync::MessagePublisher
-      p_klass.publish_model(self, action, custom_data: custom_data, custom_headers: custom_headers)
+      p_klass.publish_model(self, action, data: data, mapping: mapping, headers: headers, as_klass: as_klass)
     end
+    delegate :ps_class_publish, to: :class
 
     module ClassMethods
-      # Permit to configure to publish crud actions (:create, :update, :destroy)
-      # @param headers (Hash, optional): Refer Payload.headers
-      def ps_publish(attrs, actions: %i[create update destroy], as_klass: nil, headers: {})
-        klass = PubSubModelSync::Publisher
-        publisher = klass.new(attrs, name, actions, as_klass: as_klass, headers: headers)
-        PubSubModelSync::Config.publishers << publisher
-        actions.each do |action|
-          ps_register_callback(action.to_sym)
-        end
-      end
-
-      # Klass level notification
-      # @deprecated this method was deprecated in favor of:
-      #   PubSubModelSync::MessagePublisher.publish_data(...)
+      # Publishes a class level notification via pubsub
+      # @param data (Hash): Data of the notification
+      # @param action (Symbol): action  name of the notification
+      # @param as_klass (String, default current class name): Class name of the notification
+      # @param headers (Hash, optional): header settings (More in Payload.headers)
       def ps_class_publish(data, action:, as_klass: nil, headers: {})
         klass = PubSubModelSync::MessagePublisher
         klass.publish_data((as_klass || name).to_s, data, action.to_sym, headers: headers)
       end
 
-      # Publisher info for specific action
-      def ps_publisher(action = :create)
-        PubSubModelSync::Config.publishers.find do |publisher|
-          publisher.klass == name && publisher.actions.include?(action)
+      # @param crud_actions (Symbol|Array<Symbol>): :create, :update, :destroy
+      # @param method_name (Symbol, optional) method to be called
+      def ps_on_crud_event(crud_actions, method_name = nil, &block)
+        crud_actions = Array(crud_actions)
+        callback = ->(action) { method_name ? send(method_name, action) : instance_exec(action, &block) }
+        commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
+        crud_actions.each do |action|
+          send(commit_name, on: :create) { instance_exec(action, &callback) } if action == :create
+          send(commit_name, on: :update) { instance_exec(action, &callback) } if action == :update
+          after_destroy { instance_exec(action, &callback) } if action == :destroy
         end
       end
 
       private
 
-      # TODO: skip all enqueued notifications after_rollback (when failed)
       # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
       def ps_init_transaction_callbacks
         start_transaction = lambda do
-          key = PubSubModelSync::Publisher.ordering_key_for(self)
-          @ps_old_transaction_key = PubSubModelSync::MessagePublisher.init_transaction(key)
+          key = id ? PubSubModelSync::Publisher.ordering_key_for(self) : nil
+          @ps_transaction = PubSubModelSync::MessagePublisher.init_transaction(key)
         end
-        end_transaction = -> { PubSubModelSync::MessagePublisher.end_transaction(@ps_old_transaction_key) }
         after_create start_transaction, prepend: true # wait for ID
         before_update start_transaction, prepend: true
         before_destroy start_transaction, prepend: true
-        after_commit end_transaction
-        after_rollback end_transaction
-      end
-
-      # Configure specific callback and execute publisher when called callback
-      def ps_register_callback(action)
-        after_commit(on: action) do |model|
-          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
-          if !disabled && !model.ps_skip_callback?(action)
-            klass = PubSubModelSync::MessagePublisher
-            klass.publish_model(model, action.to_sym)
-          end
-        end
+        after_commit { @ps_transaction.deliver_all }
+        after_rollback(prepend: true) { @ps_transaction.rollback }
       end
     end
   end