pub_sub_model_sync 0.4.2.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 30ddd2acf4ff439f66a35b747794b606dbe7949c31fe3f9bf064e87e718c8e48
-  data.tar.gz: 95724a2190dd0cdb8d9a4ef4e841c90462cd53794dbf5cde7c04e2699171f55a
+  metadata.gz: 0eca4a27d7aca148acabc3412b1d56fdd3648f87d03dfb61337fee2cadea57c2
+  data.tar.gz: aea755f974264fb6e223e8680392bdc729cd917be1e93532965450f07c9a59b4
 SHA512:
-  metadata.gz: 417260220e8c7c89a0fd37a23814d4b5436f29218bd212541b11e9ec1f815cb283fc8f1a1d579dd23ee5ada60c8138cc6a53515ddea2a0ec7dd75c5ee59c2be3
-  data.tar.gz: ded28709d868ec234553ac1d47927d48423f909126fc1a4b119d5cb98801181927883ae1b0b7e01c979b0225bafbd6685f2f7b9d843722da673e3d63760b8057
+  metadata.gz: 64b6f9add9e7d382f0e3e5dae8195fdbf5e642008c7330f3e47fd55cb7f7749e381ea5df2b8d10915b907f3c6932fffc4a63a3dcfcabf88d0737a91d92c03bc7
+  data.tar.gz: edeb6931f1bdf09eebba01a036261571d8b51429475af49423f8c4de74e9241e45b59e2742e3d82b514d56f5abe4dbb503849915cfc86beca5b55b8d5a042873

data/.github/workflows/ruby.yml

@@ -42,9 +42,14 @@ jobs:
           gem install bundler -v "~> $bundler_v"
           bundle _${bundler_v}_ install --jobs 4 --retry 3
 
+#      remote ssh debugger
+#      - name: Setup tmate session (remote session debugger)
+#        uses: mxschmitt/action-tmate@v3
+
       - name: Tests (rspec)
         run: |
           bundle exec rspec
 
       - name: Code style (Rubocop)
-        run: bundle exec rubocop
+        run: bundle exec rubocop
+        if: matrix.ruby == '2.6' && matrix.rails == '6'

data/.rubocop.yml

@@ -1,7 +1,7 @@
 # This is the configuration used to check the rubocop source code.
 
 AllCops:
-  TargetRubyVersion: 2.4
+  TargetRubyVersion: 2.6
   Exclude:
     - 'spec/spec_helper.rb'
     - 'Gemfile'
@@ -13,12 +13,15 @@ Metrics/BlockLength:
     - 'spec/**/*.rb'
 
 Layout/LineLength:
-  Max: 80
+  Max: 120
 
 Style/SymbolArray:
   Exclude:
     - 'Gemfile'
 
+Lint/MissingSuper:
+  Enabled: false
+
 Style/Documentation:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Change Log
 
+# 0.5.2 (December 30, 2020)
+- fix: rabbitmq deliver messages to all subscribers
+- fix: rabbitmq persist messages to recover after restarting
+
+# 0.5.1.1 (December 29, 2020)
+- Hotfix: auto convert class name into string
+
+# 0.5.1 (December 24, 2020)
+- feat: rename publisher callbacks to be more understandable
+- feat: add callbacks to listen when processing a message (before saving sync)
+
+# 0.5.0.1 (December 22, 2020)
+- fix: add missing rabbit mock method
+
+# 0.5.0 (December 22, 2020)
+- feat: add :publish! and :process! methods to payloads
+- feat: add ability to disable publisher globally
+- fix: skip notifications from the same application
+- fix: rabbitmq use fanout instead of queue to deliver messages to multiple apps
+- refactor: include payload object to carry message info
+- feat: include notification events (when publishing and when processing messages)
+
+# 0.4.2.2 (November 29, 2020, deleted cause of typo)
+- feat: rabbitMQ skip receiving messages from the same app
+- feat: rabbitmq use fanout instead of queue to deliver messages to multiple apps
+ 
 # 0.4.2.1 (August 20, 2020)
 - Improve ```ps_subscriber_changed?``` to run validations and check for changes
  

data/Gemfile

@@ -1,6 +1,6 @@
 source "https://rubygems.org"
 
-gem 'rubocop'
+gem 'rubocop', '~> 1.6.0', require: false
 gem 'bunny' # rabbit-mq
 gem 'google-cloud-pubsub' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.4.2.1)
+    pub_sub_model_sync (0.5.2)
       rails
 
 GEM
@@ -65,7 +65,7 @@ GEM
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     amq-protocol (2.3.0)
-    ast (2.4.0)
+    ast (2.4.1)
     builder (3.2.4)
     bunny (2.14.3)
       amq-protocol (~> 2.3, >= 2.3.0)
@@ -77,7 +77,7 @@ GEM
       database_cleaner (~> 1.8.0)
     diff-lcs (1.3)
     digest-crc (0.5.1)
-    erubi (1.9.0)
+    erubi (1.10.0)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     globalid (0.4.2)
@@ -120,9 +120,8 @@ GEM
       grpc (~> 1.0)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    jaro_winkler (1.5.4)
     jwt (2.2.1)
-    loofah (2.6.0)
+    loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -137,13 +136,13 @@ GEM
     minitest (5.14.0)
     multi_json (1.14.1)
     multipart-post (2.1.1)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     os (1.0.1)
-    parallel (1.19.1)
-    parser (2.7.0.4)
-      ast (~> 2.4.0)
+    parallel (1.20.1)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
     public_suffix (4.0.3)
     rack (2.2.3)
     rack-test (1.1.0)
@@ -176,6 +175,7 @@ GEM
       thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
+    regexp_parser (2.0.1)
     rexml (3.2.4)
     rly (0.2.3)
     rspec (3.9.0)
@@ -191,14 +191,17 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-support (3.9.2)
-    rubocop (0.80.1)
-      jaro_winkler (~> 1.5.1)
+    rubocop (1.6.1)
       parallel (~> 1.10)
-      parser (>= 2.7.0.1)
+      parser (>= 2.7.1.5)
       rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
       rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 1.4.0, < 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.3.0)
+      parser (>= 2.7.1.5)
     ruby-kafka (1.0.0)
       digest-crc
     ruby-progressbar (1.10.1)
@@ -210,7 +213,7 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)
@@ -219,7 +222,7 @@ GEM
     thread_safe (0.3.6)
     tzinfo (1.2.7)
       thread_safe (~> 0.1)
-    unicode-display_width (1.6.1)
+    unicode-display_width (1.7.0)
     websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -236,7 +239,7 @@ DEPENDENCIES
   pub_sub_model_sync!
   rake
   rspec
-  rubocop
+  rubocop (~> 1.6.0)
   ruby-kafka
   sqlite3
 

data/README.md

@@ -59,7 +59,13 @@ And then execute: $ bundle install
     ```ruby
     rake pub_sub_model_sync:start
     ```
-    Note: Publishers do not need todo this
+    Note: Publishers do not need todo this    
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    ```ruby 
+      # PubSubModelSync::Config.subscribers ==> []
+      Rails.application.try(:eager_load!)
+      # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
+    ``` 
 
 - Check the service status with:    
   ```PubSubModelSync::MessagePublisher.publish_data('Test message', {sample_value: 10}, :create)```
@@ -163,6 +169,9 @@ Note: Be careful with collision of names
   ```.ps_subscriber_changed?(data)```    
   By default: ```model.changed?```
 
+- Permit to perform custom actions before saving sync of the model (:cancel can be returned to skip sync)   
+  ```.ps_before_save_sync(payload)```    
+
 ### Publishers
 - Permit to configure crud publishers
   ```ps_publish(attrs, actions: nil, as_klass: nil)```
@@ -198,11 +207,11 @@ Note: Be careful with collision of names
   * action_name: (required, :sim) Action name    
   * as_klass: (optional, :string) Custom class name (Default current model name)
       
-- Publish a class level notification (Same as above: on demand call)    
-  ```PubSubModelSync::MessagePublisher.publish_data(Klass_name, data, action_name)```  
-  * klass_name: (required, Class) same class name as defined in ps_class_subscribe(...)
-  * data: (required, :hash) message value to deliver    
-  * action_name: (required, :sim) same action name as defined in ps_class_subscribe(...)
+- Publish a class level notification (Same as above: manual call)    
+  ```ruby
+    payload = PubSubModelSync::Payload.new({ title: 'hello' }, { action: :greeting, klass: 'User' })
+    payload.publish!
+  ```
   
 - Get crud publisher configured for the class   
   ```User.ps_publisher(action_name)```  
@@ -238,27 +247,23 @@ Note: Be careful with collision of names
     ```ruby
     # Subscriber
     it 'receive model message' do
-      action = :create
       data = { name: 'name', id: 999 }
-      publisher = PubSubModelSync::MessageProcessor.new(data, 'User', action)
-      publisher.process
+      payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: :create })
+      payload.process!
       expect(User.where(id: data[:id]).any?).to be_truth
     end
       
     it 'receive class message' do
-      action = :greeting
       data = { msg: 'hello' }
-      publisher = PubSubModelSync::MessageProcessor.new(data, 'User', action)
-      publisher.process
+      action = :greeting
+      payload = PubSubModelSync::Payload.new(data, { klass: 'User', action: action })
+      payload.process!
       expect(User).to receive(action)
     end
   
     # Publisher
     it 'publish model action' do
       publisher = PubSubModelSync::MessagePublisher  
-      data = { name: 'hello'}
-      action = :create
-      User.ps_class_publish(data, action: action)
       user = User.create(name: 'name', email: 'email')
       expect(publisher).to receive(:publish_model).with(user, :create, anything)
     end
@@ -272,11 +277,36 @@ Note: Be careful with collision of names
     end
     ```
 
+## Extra configurations
+```ruby
+config = PubSubModelSync::Config
+config.debug = true
+```
+
+- ```debug = true```    
+    (true/false*) => show advanced log messages
+- ```logger = Rails.logger```   
+    (Logger) => define custom logger
+- ```disabled_callback_publisher = ->(_model, _action) { false }```   
+    (true/false*) => if true, does not listen model callbacks for auto sync (Create/Update/Destroy) 
+- ```on_before_processing = ->(payload, subscriber) { puts payload }```    
+    (Proc) => called before processing received message (:cancel can be returned to skip processing)   
+- ```on_success_processing = ->(payload, subscriber) { puts payload }```    
+    (Proc) => called when a message was successfully processed
+- ```on_error_processing = ->(exception, payload) { sleep 1; payload.process! }```    
+    (Proc) => called when a message failed when processing
+- ```on_before_publish = ->(payload) { puts payload }```    
+    (Proc) => called before publishing a message (:cancel can be returned to skip publishing)    
+- ```on_after_publish = ->(payload) { puts payload }```    
+    (Proc) => called after publishing a message
+- ```on_error_publish = ->(exception, payload) { sleep 1; payload.publish! }```    
+    (Proc) => called when failed publishing a message
+    
 ## TODO
-- Hooks/callbacks when message processed or failed
 - Add alias attributes when subscribing (similar to publisher)
 - Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
- 
+- Auto publish update only if payload has changed
+- On delete, payload must only be composed by ids 
 
 ## Contributing
 

data/lib/pub_sub_model_sync.rb

@@ -5,6 +5,7 @@ require 'active_support'
 
 require 'pub_sub_model_sync/railtie'
 require 'pub_sub_model_sync/config'
+require 'pub_sub_model_sync/base'
 require 'pub_sub_model_sync/subscriber_concern'
 require 'pub_sub_model_sync/message_publisher'
 require 'pub_sub_model_sync/publisher_concern'

data/lib/pub_sub_model_sync/base.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Base
+    delegate :config, :log, to: self
+
+    class << self
+      def config
+        PubSubModelSync::Config
+      end
+
+      def log(message, kind = :info)
+        config.log message, kind
+      end
+    end
+  end
+end

data/lib/pub_sub_model_sync/config.rb

@@ -5,16 +5,27 @@ module PubSubModelSync
     cattr_accessor(:subscribers) { [] }
     cattr_accessor(:publishers) { [] }
     cattr_accessor(:service_name) { :google }
-    cattr_accessor :logger
+
+    # customizable callbacks
+    cattr_accessor(:debug) { false }
+    cattr_accessor :logger # LoggerInst
+
+    cattr_accessor(:on_before_processing) { ->(_payload, _subscriber) {} } # return :cancel to skip
+    cattr_accessor(:on_success_processing) { ->(_payload, _subscriber) {} }
+    cattr_accessor(:on_error_processing) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
+    cattr_accessor(:on_after_publish) { ->(_payload) {} }
+    cattr_accessor(:on_error_publish) { ->(_exception, _payload) {} }
+    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
     cattr_accessor :project, :credentials, :topic_name, :subscription_name
 
     # rabbitmq service
-    cattr_accessor :bunny_connection, :queue_name, :topic_name
+    cattr_accessor :bunny_connection, :queue_name, :topic_name, :subscription_name
 
     # kafka service
-    cattr_accessor :kafka_connection, :topic_name
+    cattr_accessor :kafka_connection, :topic_name, :subscription_name
 
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
@@ -24,5 +35,10 @@ module PubSubModelSync
         logger ? logger.send(kind, msg) : puts(msg)
       end
     end
+
+    def self.subscription_key
+      subscription_name ||
+        (Rails.application.class.parent_name rescue '') # rubocop:disable Style/RescueModifier
+    end
   end
 end

data/lib/pub_sub_model_sync/connector.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Connector
     attr_accessor :service
+
     delegate :listen_messages, :publish, :stop, to: :service
 
     def initialize

data/lib/pub_sub_model_sync/message_processor.rb

@@ -1,40 +1,54 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class MessageProcessor
-    attr_accessor :data, :klass, :action
-
-    # @param data (Hash): any hash value to deliver
-    def initialize(data, klass, action)
-      @data = data
-      @klass = klass
-      @action = action
+  class MessageProcessor < PubSubModelSync::Base
+    attr_accessor :payload
+
+    # @param payload (Payload): payload to be delivered
+    # @Deprecated: def initialize(data, klass, action)
+    def initialize(payload, klass = nil, action = nil)
+      @payload = payload
+      return if @payload.is_a?(Payload)
+
+      # support for deprecated
+      log('Deprecated: Use Payload instead of new(data, klass, action)')
+      @payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
     end
 
     def process
-      subscribers = filter_subscribers
-      subscribers.each { |subscriber| run_subscriber(subscriber) }
+      filter_subscribers.each(&method(:run_subscriber))
     end
 
     private
 
     def run_subscriber(subscriber)
-      subscriber.eval_message(data)
-      log "processed message with: #{[klass, action, data]}"
+      return unless processable?(subscriber)
+
+      subscriber.process!(payload)
+      res = config.on_success_processing.call(payload, subscriber)
+      log "processed message with: #{payload.inspect}" if res != :skip_log
     rescue => e
-      info = [klass, action, data, e.message, e.backtrace]
-      log("error processing message: #{info}", :error)
+      print_subscriber_error(e)
     end
 
-    def filter_subscribers
-      PubSubModelSync::Config.subscribers.select do |subscriber|
-        subscriber.settings[:from_klass].to_s == klass.to_s &&
-          subscriber.settings[:from_action].to_s == action.to_s
-      end
+    def processable?(subscriber)
+      cancel = config.on_before_processing.call(payload, subscriber) == :cancel
+      log("process message cancelled: #{payload}") if cancel && config.debug
+      !cancel
     end
 
-    def log(message, kind = :info)
-      PubSubModelSync::Config.log message, kind
+    # @param error (Error)
+    def print_subscriber_error(error)
+      info = [payload, error.message, error.backtrace]
+      res = config.on_error_processing.call(error, payload)
+      log("Error processing message: #{info}", :error) if res != :skip_log
+    end
+
+    def filter_subscribers
+      config.subscribers.select do |subscriber|
+        subscriber.settings[:from_klass].to_s == payload.klass.to_s &&
+          subscriber.settings[:from_action].to_s == payload.action.to_s
+      end
     end
   end
 end

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -1,17 +1,15 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  class MessagePublisher
+  class MessagePublisher < PubSubModelSync::Base
     class << self
-      delegate :publish, to: :connector
-
       def connector
         @connector ||= PubSubModelSync::Connector.new
       end
 
       def publish_data(klass, data, action)
-        attrs = { klass: klass.to_s, action: action.to_sym }
-        publish(data, attrs)
+        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
+        publish(payload)
       end
 
       # @param model: ActiveRecord model
@@ -22,11 +20,32 @@ module PubSubModelSync
 
         publisher ||= model.class.ps_publisher(action)
         payload = publisher.payload(model, action)
-        res_before = model.ps_before_sync(action, payload[:data])
+        res_before = model.ps_before_sync(action, payload.data)
         return if res_before == :cancel
 
-        publish(payload[:data], payload[:attrs])
-        model.ps_after_sync(action, payload[:data])
+        publish(payload)
+        model.ps_after_sync(action, payload.data)
+      end
+
+      def publish(payload)
+        if config.on_before_publish.call(payload) == :cancel
+          log("Publish message cancelled: #{payload}") if config.debug
+          return
+        end
+
+        log("Publishing message: #{[payload]}")
+        connector.publish(payload)
+        config.on_after_publish.call(payload)
+      rescue => e
+        notify_error(e, payload)
+      end
+
+      private
+
+      def notify_error(exception, payload)
+        info = [payload, exception.message, exception.backtrace]
+        res = config.on_error_publish.call(exception, payload)
+        log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end
   end

data/lib/pub_sub_model_sync/mock_rabbit_service.rb

@@ -20,12 +20,17 @@ module PubSubModelSync
       def name
         'name'
       end
+
+      def publish(*_args)
+        true
+      end
     end
 
     class MockChannel
       def queue(*_args)
         @queue ||= MockQueue.new
       end
+      alias fanout queue
 
       def topic(*_args)
         @topic ||= MockTopic.new

data/lib/pub_sub_model_sync/payload.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class Payload
+    attr_reader :data, :attributes, :headers
+
+    # @param data (Hash: { any value }):
+    # @param attributes (Hash: { klass: string, action: :sym }):
+    def initialize(data, attributes, headers = {})
+      @data = data
+      @attributes = attributes
+      @headers = headers
+      build_headers
+    end
+
+    def to_h
+      { data: data, attributes: attributes, headers: headers }
+    end
+
+    def klass
+      attributes[:klass]
+    end
+
+    def action
+      attributes[:action]
+    end
+
+    def process!
+      publisher = PubSubModelSync::MessageProcessor.new(self)
+      publisher.process
+    end
+
+    def publish!
+      klass = PubSubModelSync::MessagePublisher
+      klass.publish(self)
+    end
+
+    private
+
+    def build_headers
+      headers[:uuid] ||= SecureRandom.uuid
+      headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+    end
+  end
+end

data/lib/pub_sub_model_sync/publisher.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Publisher
     attr_accessor :attrs, :actions, :klass, :as_klass
+
     def initialize(attrs, klass, actions = nil, as_klass = nil)
       @attrs = attrs
       @klass = klass
@@ -11,7 +12,7 @@ module PubSubModelSync
     end
 
     def payload(model, action)
-      { data: payload_data(model), attrs: payload_attrs(model, action) }
+      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
     end
 
     private

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -11,13 +11,12 @@ module PubSubModelSync
       false
     end
 
-    # TODO: make it using respond_to?(:ps_skip_sync?)
     # before preparing data to sync
     def ps_skip_sync?(_action)
       false
     end
 
-    # before delivering data
+    # before delivering data (return :cancel to cancel sync)
     def ps_before_sync(_action, _data); end
 
     # after delivering data
@@ -64,7 +63,8 @@ module PubSubModelSync
 
       def ps_register_callback(action, publisher)
         after_commit(on: action) do |model|
-          unless model.ps_skip_callback?(action)
+          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
+          if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
             klass.publish_model(model, action.to_sym, publisher)
           end

data/lib/pub_sub_model_sync/service_base.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
+require 'pub_sub_model_sync/payload'
 module PubSubModelSync
-  class ServiceBase
+  class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'
     end
 
-    def publish(_data, _attributes)
+    # @param _payload (Payload)
+    def publish(_payload)
       raise 'method :publish must be defined in service'
     end
 
@@ -18,19 +20,29 @@ module PubSubModelSync
 
     private
 
-    # @param payload (String JSON): '{"data":{}, "attributes":{..}}'
-    #   refer: PubSubModelSync::MessagePublisher(.publish_model | .publish_data)
-    def perform_message(payload)
-      data, attrs = parse_message_payload(payload)
-      args = [data, attrs[:klass], attrs[:action]]
-      PubSubModelSync::MessageProcessor.new(*args).process
+    # @param (String: Payload in json format)
+    def process_message(payload_info)
+      payload = parse_payload(payload_info)
+      log("Received message: #{[payload]}") if config.debug
+      if same_app_message?(payload)
+        log("Skip message from same origin: #{[payload]}") if config.debug
+      else
+        payload.process!
+      end
+    rescue => e
+      error = [payload, e.message, e.backtrace]
+      log("Error parsing received message: #{error}", :error)
     end
 
-    def parse_message_payload(payload)
-      message_payload = JSON.parse(payload).symbolize_keys
-      data = message_payload[:data].symbolize_keys
-      attrs = message_payload[:attributes].symbolize_keys
-      [data, attrs]
+    def parse_payload(payload_info)
+      info = JSON.parse(payload_info).deep_symbolize_keys
+      ::PubSubModelSync::Payload.new(info[:data], info[:attributes], info[:headers])
+    end
+
+    # @param payload (Payload)
+    def same_app_message?(payload)
+      key = payload.headers[:app_key]
+      key && key == config.subscription_key
     end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,10 +7,9 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    attr_accessor :service, :topic, :subscription, :config, :subscriber
+    attr_accessor :service, :topic, :subscription, :subscriber
 
     def initialize
-      @config = PubSubModelSync::Config
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
       @topic = service.topic(config.topic_name) ||
@@ -28,13 +27,8 @@ module PubSubModelSync
       log('Listener stopped')
     end
 
-    def publish(data, attributes)
-      log("Publishing message: #{[attributes, data]}")
-      payload = { data: data, attributes: attributes }.to_json
-      topic.publish(payload, { SERVICE_KEY => true })
-    rescue => e
-      info = [attributes, data, e.message, e.backtrace]
-      log("Error publishing: #{info}", :error)
+    def publish(payload)
+      topic.publish(payload.to_json, { SERVICE_KEY => true })
     end
 
     def stop
@@ -51,17 +45,9 @@ module PubSubModelSync
 
     def process_message(received_message)
       message = received_message.message
-      return unless message.attributes[SERVICE_KEY]
-
-      perform_message(message.data)
-    rescue => e
-      log("Error processing message: #{[received_message, e.message]}", :error)
+      super(message.data) if message.attributes[SERVICE_KEY]
     ensure
       received_message.acknowledge!
     end
-
-    def log(msg, kind = :info)
-      config.log("Google Service ==> #{msg}", kind)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -8,9 +8,8 @@ end
 module PubSubModelSync
   class ServiceKafka < ServiceBase
     cattr_accessor :producer
+    attr_accessor :config, :service, :consumer
 
-    attr_accessor :service, :consumer
-    attr_accessor :config
     CONSUMER_GROUP = 'service_model_sync'
 
     def initialize
@@ -23,19 +22,14 @@ module PubSubModelSync
       start_consumer
       consumer.each_message(&method(:process_message))
     rescue PubSubModelSync::Runner::ShutDown
-      raise
+      log('Listener stopped')
     rescue => e
       log("Error listening message: #{[e.message, e.backtrace]}", :error)
     end
 
-    def publish(data, attributes)
-      log("Publishing: #{[attributes, data]}")
-      payload = { data: data, attributes: attributes }
+    def publish(payload)
       producer.produce(payload.to_json, message_settings)
       producer.deliver_messages
-    rescue => e
-      info = [attributes, data, e.message, e.backtrace]
-      log("Error publishing: #{info}", :error)
     end
 
     def stop
@@ -64,14 +58,7 @@ module PubSubModelSync
     def process_message(message)
       return unless message.headers[SERVICE_KEY]
 
-      perform_message(message.value)
-    rescue => e
-      error = [message, e.message, e.backtrace]
-      log("Error processing message: #{error}", :error)
-    end
-
-    def log(msg, kind = :info)
-      config.log("Kafka Service ==> #{msg}", kind)
+      super(message.value)
     end
   end
 end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -7,8 +7,7 @@ end
 
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
-    attr_accessor :service, :channel, :queue, :topic
-    attr_accessor :config
+    attr_accessor :config, :service, :channel, :queue, :topic
 
     def initialize
       @config = PubSubModelSync::Config
@@ -22,33 +21,41 @@ module PubSubModelSync
       queue.subscribe(subscribe_settings, &method(:process_message))
       loop { sleep 5 }
     rescue PubSubModelSync::Runner::ShutDown
-      raise
+      log('Listener stopped')
     rescue => e
       log("Error listening message: #{[e.message, e.backtrace]}", :error)
     end
 
-    def publish(data, attributes)
-      log("Publishing: #{[attributes, data]}")
-      deliver_data(data, attributes)
-    # TODO: max retry
-    rescue Timeout::Error => e
-      log("Error publishing (retrying....): #{e.message}", :error)
-      initialize
-      retry
+    def publish(payload)
+      qty_retry ||= 0
+      deliver_data(payload)
     rescue => e
-      info = [attributes, data, e.message, e.backtrace]
-      log("Error publishing: #{info}", :error)
+      if e.is_a?(Timeout::Error) && (qty_retry += 1) <= 2
+        log("Error publishing (retrying....): #{e.message}", :error)
+        initialize
+        retry
+      end
+      raise
     end
 
     def stop
       log('Listener stopping...')
+      channel&.close
       service.close
     end
 
     private
 
     def message_settings
-      { routing_key: queue.name, type: SERVICE_KEY }
+      {
+        routing_key: queue.name,
+        type: SERVICE_KEY,
+        persistent: true
+      }
+    end
+
+    def queue_settings
+      { durable: true, auto_delete: false }
     end
 
     def subscribe_settings
@@ -58,32 +65,23 @@ module PubSubModelSync
     def process_message(_delivery_info, meta_info, payload)
       return unless meta_info[:type] == SERVICE_KEY
 
-      perform_message(payload)
-    rescue => e
-      error = [payload, e.message, e.backtrace]
-      log("Error processing message: #{error}", :error)
+      super(payload)
     end
 
     def subscribe_to_queue
       service.start
       @channel = service.create_channel
-      queue_settings = { durable: true, auto_delete: false }
-      @queue = channel.queue(config.queue_name, queue_settings)
-      subscribe_to_topic
+      @queue = channel.queue(config.subscription_key, queue_settings)
+      subscribe_to_exchange
     end
 
-    def subscribe_to_topic
-      @topic = channel.topic(config.topic_name)
+    def subscribe_to_exchange
+      @topic = channel.fanout(config.topic_name)
       queue.bind(topic, routing_key: queue.name)
     end
 
-    def log(msg, kind = :info)
-      config.log("Rabbit Service ==> #{msg}", kind)
-    end
-
-    def deliver_data(data, attributes)
+    def deliver_data(payload)
       subscribe_to_queue
-      payload = { data: data, attributes: attributes }
       topic.publish(payload.to_json, message_settings)
 
       # Ugly fix: "IO timeout when reading 7 bytes"

data/lib/pub_sub_model_sync/subscriber.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Subscriber
     attr_accessor :klass, :action, :attrs, :settings
+    attr_reader :payload
 
     # @param settings: (Hash) { id: :id, direct_mode: false,
     #                           from_klass: klass, from_action: action }
@@ -15,50 +16,51 @@ module PubSubModelSync
       @settings = def_settings.merge(settings)
     end
 
-    def eval_message(message)
+    def process!(payload)
+      @payload = payload
       if settings[:direct_mode]
-        run_class_message(message)
+        run_class_message
       else
-        run_model_message(message)
+        run_model_message
       end
     end
 
     private
 
-    def run_class_message(message)
+    def run_class_message
       model_class = klass.constantize
-      model_class.send(action, message)
+      model_class.send(action, payload.data)
     end
 
     # support for: create, update, destroy
-    def run_model_message(message)
-      model = find_model(message)
+    def run_model_message
+      model = find_model
+      return if model.ps_before_save_sync(payload) == :cancel
+
       if action == :destroy
         model.destroy!
       else
-        populate_model(model, message)
-        return if action == :update && !model.ps_subscriber_changed?(message)
+        populate_model(model)
+        return if action == :update && !model.ps_subscriber_changed?(payload.data)
 
         model.save!
       end
     end
 
-    def find_model(message)
+    def find_model
       model_class = klass.constantize
-      if model_class.respond_to?(:ps_find_model)
-        return model_class.ps_find_model(message)
-      end
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
 
-      model_class.where(model_identifiers(message)).first_or_initialize
+      model_class.where(model_identifiers).first_or_initialize
     end
 
-    def model_identifiers(message)
+    def model_identifiers
       identifiers = Array(settings[:id])
-      identifiers.map { |key| [key, message[key.to_sym]] }.to_h
+      identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
     end
 
-    def populate_model(model, message)
-      values = message.slice(*attrs)
+    def populate_model(model)
+      values = payload.data.slice(*attrs)
       values.each do |attr, value|
         model.send("#{attr}=", value)
       end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -12,6 +12,10 @@ module PubSubModelSync
       changed?
     end
 
+    # permit to apply custom actions before applying sync
+    # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
+    def ps_before_save_sync(_payload); end
+
     module ClassMethods
       def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
         settings = { id: id, from_klass: from_klass }

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.4.2.1'
+  VERSION = '0.5.2'
 end

data/pub_sub_model_sync.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'pub_sub_model_sync/version'
 
 Gem::Specification.new do |spec|
-  spec.required_ruby_version = '>= 2.4'
+  spec.required_ruby_version = '>= 2.4' # rubocop:disable Gemspec/RequiredRubyVersion
   spec.name          = 'pub_sub_model_sync'
   spec.version       = PubSubModelSync::VERSION
   spec.authors       = ['Owen']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.4.2.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-20 00:00:00.000000000 Z
+date: 2020-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -104,6 +104,7 @@ files:
 - gemfiles/Gemfile_5
 - gemfiles/Gemfile_6
 - lib/pub_sub_model_sync.rb
+- lib/pub_sub_model_sync/base.rb
 - lib/pub_sub_model_sync/config.rb
 - lib/pub_sub_model_sync/connector.rb
 - lib/pub_sub_model_sync/message_processor.rb
@@ -111,6 +112,7 @@ files:
 - lib/pub_sub_model_sync/mock_google_service.rb
 - lib/pub_sub_model_sync/mock_kafka_service.rb
 - lib/pub_sub_model_sync/mock_rabbit_service.rb
+- lib/pub_sub_model_sync/payload.rb
 - lib/pub_sub_model_sync/publisher.rb
 - lib/pub_sub_model_sync/publisher_concern.rb
 - lib/pub_sub_model_sync/railtie.rb