pub_sub_model_sync 1.0.beta → 1.0.1

data/lib/pub_sub_model_sync/railtie.rb

@@ -2,6 +2,8 @@
 
 require 'pub_sub_model_sync'
 require 'rails'
+require 'active_record'
+require 'pub_sub_model_sync/config'
 module PubSubModelSync
   class Railtie < ::Rails::Railtie
     railtie_name :pub_sub_model_sync
@@ -9,5 +11,9 @@ module PubSubModelSync
     rake_tasks do
       load 'pub_sub_model_sync/tasks/worker.rake'
     end
+
+    configure do
+      require 'pub_sub_model_sync/initializers/before_commit' if PubSubModelSync::Config.enable_rails4_before_commit
+    end
   end
 end

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -26,7 +26,7 @@ module PubSubModelSync
     def run_class_message
       model_class = subscriber.klass.constantize
       model_class.ps_processing_payload = payload # TODO: review for parallel notifications
-      call_action(model_class, payload.data) if ensure_sync(model_class)
+      call_action(model_class) if ensure_sync(model_class)
     end
 
     # support for: create, update, destroy
@@ -48,16 +48,17 @@ module PubSubModelSync
       res
     end
 
-    def call_action(object, *args)
-      action_name = settings[:to_action]
-      if action_name.is_a?(Proc)
-        args.prepend(object) unless klass_subscription?
-        action_name.call(*args)
-      else # method name
-        action_name = :save if %i[create update].include?(action_name.to_sym)
-        object.send(action_name, *args)
-      end
-      raise(object.errors) if object.respond_to?(:errors) && object.errors.any?
+    def call_action(object)
+      callback = settings[:to_action]
+      callback.is_a?(Proc) ? object.instance_exec(payload.data, &callback) : call_action_method(object)
+    end
+
+    def call_action_method(object)
+      method_name = settings[:to_action]
+      method_name = :save! if %i[create update].include?(method_name.to_sym)
+      method_name = :destroy! if method_name.to_sym == :destroy
+      is_crud_action = %i[save! destroy!].include?(method_name)
+      is_crud_action ? object.send(method_name) : object.send(method_name, payload.data)
     end
 
     def parse_condition(condition, object)
@@ -74,10 +75,13 @@ module PubSubModelSync
       model_class = subscriber.klass.constantize
       return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
 
+      error_msg = 'No values provided for identifiers:'
+      raise(StandardError, "#{error_msg} #{[settings[:id], payload]}") if model_identifiers.empty?
+
       model_class.where(model_identifiers).first_or_initialize
     end
 
-    # @param mappings (Array<String>) supports aliasing, sample: ["id", "full_name:name"]
+    # @param mappings (Array<String,Symbol>) supports aliasing, sample: ["id", "full_name:name"]
     # @return (Hash) hash with the correct attr names and its values
     def parse_mapping(mappings)
       mappings.map do |prop|
@@ -91,7 +95,7 @@ module PubSubModelSync
 
     # @return (Hash) hash including identifiers and its values
     def model_identifiers
-      @model_identifiers ||= parse_mapping(Array(settings[:id]).map(&:to_s))
+      @model_identifiers ||= parse_mapping(Array(settings[:id]))
     end
 
     def populate_model

data/lib/pub_sub_model_sync/runner.rb

@@ -3,7 +3,6 @@
 require 'active_support/core_ext/module'
 module PubSubModelSync
   class Runner
-    class ShutDown < StandardError; end
     delegate :preload_listeners, to: :class
     attr_accessor :connector
 
@@ -12,11 +11,10 @@ module PubSubModelSync
     end
 
     def run
+      at_exit { connector.stop }
       trap_signals!
       preload_listeners
       start_listeners
-    rescue ShutDown
-      connector.stop
     end
 
     def self.preload_listeners
@@ -32,8 +30,8 @@ module PubSubModelSync
 
     def trap_signals!
       handler = proc do |signal|
-        puts "received #{Signal.signame(signal)}"
-        raise ShutDown
+        puts "PS_MSYNC ==> received #{Signal.signame(signal)}"
+        exit
       end
       %w[INT QUIT TERM].each { |signal| Signal.trap(signal, handler) }
     end

data/lib/pub_sub_model_sync/service_base.rb

@@ -6,16 +6,16 @@ module PubSubModelSync
     SERVICE_KEY = 'service_model_sync'
 
     def listen_messages
-      raise 'method :listen_messages must be defined in service'
+      raise NoMethodError, 'method :listen_messages must be defined in service'
     end
 
     # @param _payload (Payload)
     def publish(_payload)
-      raise 'method :publish must be defined in service'
+      raise NoMethodError, 'method :publish must be defined in service'
     end
 
     def stop
-      raise 'method :stop must be defined in service'
+      raise NoMethodError, 'method :stop must be defined in service'
     end
 
     private
@@ -32,26 +32,13 @@ module PubSubModelSync
 
     # @param (String: Payload in json format)
     def process_message(payload_info)
-      retries ||= 0
       payload = decode_payload(payload_info)
       return payload.process unless same_app_message?(payload)
 
       log("Skipping message from same origin: #{[payload]}") if config.debug
     rescue => e
-      retry if can_retry_process_message?(e, payload, retries += 1)
-    end
-
-    def can_retry_process_message?(error, payload, retries)
-      error_payload = [payload, error.message, error.backtrace]
-      if retries <= 5
-        sleep(retries)
-        log("Error while starting to process a message (retrying #{retries} retries...): #{error_payload}", :error)
-        rescue_database_connection if lost_db_connection_err?(error)
-        true
-      else
-        log("Retried 5 times and error persists, exiting...: #{error_payload}", :error)
-        Process.exit!(true)
-      end
+      error_payload = [payload, e.message, e.backtrace]
+      log("Error while starting to process a message: #{error_payload}", :error)
     end
 
     # @return Payload
@@ -66,19 +53,5 @@ module PubSubModelSync
       key = payload.headers[:app_key]
       key && key == config.subscription_key
     end
-
-    def lost_db_connection_err?(error)
-      return true if error.class.name == 'PG::UnableToSend' # rubocop:disable Style/ClassEqualityComparison
-
-      error.message.match?(/lost connection/i)
-    end
-
-    def rescue_database_connection
-      log('Lost DB connection. Attempting to reconnect...', :warn)
-      ActiveRecord::Base.connection.reconnect!
-    rescue
-      log('Cannot reconnect to database, exiting...', :error)
-      Process.exit!(true)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -37,14 +37,14 @@ module PubSubModelSync
       message_topics = p_topic_names.map(&method(:find_topic))
       message_topics.each do |topic|
         topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
-          raise 'Failed to publish the message.' unless res.succeeded?
+          raise StandardError, 'Failed to publish the message.' unless res.succeeded?
         end
       end
     end
 
     def stop
       log('Listener stopping...')
-      subscribers.each(&:stop!)
+      (subscribers || []).each(&:stop!)
     end
 
     private

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -76,8 +76,8 @@ module PubSubModelSync
     end
 
     # Check topic existence, create if missing topic
-    # @param names (Array<String>|String)
-    # @return (Array|String) return @param names
+    # @param names (Array<String>,String)
+    # @return (Array,String) return @param names
     def ensure_topics(names)
       missing_topics = Array(names) - (@known_topics || service.topics)
       missing_topics.each do |name|

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -8,7 +8,7 @@ end
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
     QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
-    LISTEN_SETTINGS = { manual_ack: true }.freeze
+    LISTEN_SETTINGS = { manual_ack: false }.freeze
     PUBLISH_SETTINGS = {}.freeze
 
     # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -9,25 +9,27 @@ module PubSubModelSync
     end
 
     module ClassMethods
-      # @param actions (Symbol|Array<Symbol>) Notification.action name: save|create|update|destroy|<any_other_action>
-      # @param mapping (Array<String>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
+      # @param actions (Symbol,Array<Symbol>) Notification.action name: save|create|update|destroy|<any_other_action>
+      # @param mapping (Array<String,Symbol>) Attributes mapping with aliasing support, sample: ["id", "full_name:name"]
       # @param settings (Hash<:from_klass, :to_action, :id, :if, :unless>)
       #   from_klass (String) Notification.class name
-      #   to_action (Symbol|Proc):
+      #   to_action (Symbol,Proc):
       #     Symbol: Method to process the notification
       #     Proc: Block to process the notification
-      #   id (Symbol|Array<Symbol|String>) attribute(s) DB primary identifier(s). Supports for mapping format.
-      #   if (Symbol|Proc|Array<Symbol>) Method or block called as the conformation before calling the callback
-      #   unless (Symbol|Proc|Array<Symbol>) Method or block called as the negation before calling the callback
-      def ps_subscribe(actions, mapping = [], settings = {})
-        Array(actions).each do |action|
+      #   id (Symbol,Array<Symbol,String>) attribute(s) DB primary identifier(s). Supports for mapping format.
+      #   if (Symbol,Proc,Array<Symbol>) Method or block called as the conformation before calling the callback
+      #   unless (Symbol,Proc,Array<Symbol>) Method or block called as the negation before calling the callback
+      def ps_subscribe(actions, mapping = [], settings = {}, &block)
+        settings[:to_action] ||= block if block
+        Array(actions).map do |action|
           add_ps_subscriber(action, mapping, settings)
         end
       end
 
       # @param action (Symbol) Notification.action name
       # @param settings (Hash) @refer ps_subscribe.settings except(:id)
-      def ps_class_subscribe(action, settings = {})
+      def ps_class_subscribe(action, settings = {}, &block)
+        settings[:to_action] ||= block if block
         add_ps_subscriber(action, nil, settings.merge(mode: :klass))
       end
 

data/lib/pub_sub_model_sync/transaction.rb

@@ -3,55 +3,71 @@
 module PubSubModelSync
   class Transaction < Base
     PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
-    attr_accessor :key, :payloads, :use_buffer, :parent, :children
+    attr_accessor :key, :payloads, :max_buffer, :root, :children, :finished
 
-    # @param key (String|nil) Transaction key, if empty will use the ordering_key from first payload
-    # @param use_buffer (Boolean, default: true) If false, payloads are delivered immediately
-    #   (no way to cancel/rollback if transaction failed)
-    def initialize(key, use_buffer: config.transactions_use_buffer)
+    # @param key (String,Null) Transaction key, if empty will use the ordering_key from first payload
+    # @param max_buffer (Integer) Once this quantity of notifications is reached, then all notifications
+    #   will immediately be delivered.
+    #   Note: There is no way to rollback delivered notifications if current transaction fails
+    def initialize(key, max_buffer: config.transactions_max_buffer)
       @key = key
-      @use_buffer = use_buffer
+      @max_buffer = max_buffer
       @children = []
       @payloads = []
     end
 
     # @param payload (Payload)
     def add_payload(payload)
-      use_buffer ? payloads << payload : deliver_payload(payload)
+      payloads << payload
+      log("Payload added to current transaction: #{payload.inspect}") if config.debug
+      return unless payloads.count >= max_buffer
+
+      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
+      deliver_payloads
     end
 
-    def deliver_all
-      if parent
-        parent.children = parent.children.reject { |t| t == self }
-        parent.deliver_all
+    def finish # rubocop:disable Metrics/AbcSize
+      if root
+        root.children = root.children.reject { |t| t == self }
+        root.deliver_all if root.finished && root.children.empty?
       end
-      payloads.each(&method(:deliver_payload)) if children.empty?
-      clean_publisher
+      self.finished = true
+      deliver_all if children.empty?
     end
 
     def add_transaction(transaction)
-      transaction.parent = self
+      transaction.root = self
       children << transaction
       transaction
     end
 
     def rollback
-      log("rollback #{children.count} notifications", :warn) if children.any? && debug?
+      log("Rollback #{payloads.count} notifications", :warn) if children.any? && debug?
       self.children = []
-      parent&.rollback
+      root&.rollback
       clean_publisher
     end
 
     def clean_publisher
-      PUBLISHER_KLASS.current_transaction = nil if !parent && children.empty?
+      PUBLISHER_KLASS.current_transaction = nil if !root && children.empty?
+    end
+
+    def deliver_all
+      deliver_payloads
+      clean_publisher
     end
 
     private
 
-    def deliver_payload(payload)
-      PUBLISHER_KLASS.connector_publish(payload)
-    rescue => e
-      PUBLISHER_KLASS.send(:notify_error, e, payload)
+    def deliver_payloads
+      payloads.each do |payload|
+        begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
+          PUBLISHER_KLASS.connector_publish(payload)
+        rescue => e
+          PUBLISHER_KLASS.send(:notify_error, e, payload)
+        end
+      end
+      self.payloads = []
     end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.0.beta'
+  VERSION = '1.0.1'
 end

data/samples/README.md

@@ -0,0 +1,50 @@
+# Sample model sync
+This is a sample to sync information between rails applications using RabbitMQ
+
+## Installation
+* Create manually the required network to share rabbitMQ (just if not exist):   
+  ```docker network create shared_app_services```
+  
+* Start RabbitMQ server   
+  ```cd samples/app1 && docker-compose up pubsub```
+
+* In another tab access to App1 to publish notifications (Wait for step 2)  
+  - Access to the application    
+    `cd samples/app1`    
+  
+  - Build docker and enter rails console    
+    ```docker-compose run app bash -c "rails db:migrate && rails c"```
+  
+  - Create a sample user    
+    ```ruby
+      user = User.create!(name: 'User 1', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
+    ```
+    Note: Check app2 console to see notifications (3 notifications)    
+    Note2: Access app2 console to see user and its posts
+    
+  - Update previous user
+    ```ruby
+      user.update!(name: 'User 1 changed', posts_attributes: user.posts.map { |post| { id: post.id, title: "#{post.title} changed" } })
+    ```
+    Note: Check app2 console to see notifications (3 notifications)    
+    Note2: Access app2 console to see changes for user and its posts
+    
+  - Destroy previous user
+    ```ruby
+      user.destroy!
+    ```    
+
+* In another tab access to App2 to listen notifications (Wait for step 2)      
+  - Access to the folder
+    `cd samples/app2`
+  
+  - Build docker and start listener (Received notifications will be printed here)       
+    ```docker-compose run listener```
+  
+  - Optional: Open another tab to access application to ensure synced data
+    ```docker-compose run listener bash -c "rails c```
+    ```ruby
+      user = User.last.inspect
+      user.posts.inspect
+    ```
+  

data/samples/app1/Dockerfile

@@ -0,0 +1,13 @@
+FROM ruby:2.7.1 AS builder
+
+# Allow apt to work with https-based sources
+RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends apt-transport-https
+
+RUN mkdir /app
+WORKDIR /app
+
+# backend
+COPY Gemfile.lock Gemfile /app/
+RUN gem install bundler && bundle install
+
+COPY . /app

data/samples/app1/Gemfile

@@ -0,0 +1,37 @@
+source 'https://rubygems.org'
+git_source(:github) { |repo| "https://github.com/#{repo}.git" }
+
+# Bundle edge Rails instead: gem 'rails', github: 'rails/rails', branch: 'main'
+gem 'rails', '~> 6.1.3', '>= 6.1.3.2'
+# Use sqlite3 as the database for Active Record
+gem 'sqlite3', '~> 1.4'
+# Use Puma as the app server
+gem 'puma', '~> 5.0'
+# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
+# gem 'jbuilder', '~> 2.7'
+# Use Active Model has_secure_password
+# gem 'bcrypt', '~> 3.1.7'
+
+# Reduces boot times through caching; required in config/boot.rb
+gem 'bootsnap', '>= 1.4.4', require: false
+
+# Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible
+# gem 'rack-cors'
+
+gem 'pub_sub_model_sync', '>= 1.0'
+gem 'bunny' # to use rabbit-mq pub/sub service
+gem 'annotate'
+
+group :development, :test do
+  # Call 'byebug' anywhere in the code to stop execution and get a debugger console
+  gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
+end
+
+group :development do
+  gem 'listen', '~> 3.3'
+  # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
+  gem 'spring'
+end
+
+# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
+gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]