pub_sub_model_sync 0.5.0.1 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f08249ef28730afdc2a0f3b025ef66d5117694fdc4ed995e9eb907e37e60defb
-  data.tar.gz: cc8d337de6952db6e9747912178ce7fd09f93c3a600d1d4756671fb6bcefadad
+  metadata.gz: 57e8d23c9e514734a8a45664f641378bbb46e8de5315c536a0f692efcbe29dd2
+  data.tar.gz: d0f290dac2b0563ffe5412f85e1a6a6e340e281049755b26dc732a3661501b92
 SHA512:
-  metadata.gz: cdca1e7397fa5fae46198fce8ae24036318d619d53cff5c06e8e8c266423a5681e48588eddcced3271747242a35fc77b1c75bf567d5461c4fa7f573ae29c4c15
-  data.tar.gz: 0c48d83f3904c5872cdc3b48f2d341be84cbe84899a3fdc0738c9ad290e5e6c1c49ca2ab226e12df4f097d6f927e603b4691f64760a8d4dd9c703431711e71ec
+  metadata.gz: 351901f98e6c806e7a4efe20abdac65de5bf8f716dc57e1889302eab02d33b3f2f4d449cd0d351fd5ae882d16abe79a77791bea8ad956c52cda47829319eb858
+  data.tar.gz: e906d601a8c37b999a8b6d550e605e9c04b67db692cec4b60220ae286bef9adde7132d8614cee4ca7dab9ef43538ae2baede3f07e402c4c21a189b79a9ba6873

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Change Log
 
+# 0.5.4 (January 8, 2021)
+- fix: exclude identifiers when syncing model
+- feat: callbacks support for future extra params
+- feat: make connectors configurable
+- feat: add :process! and :process, :publish!, :publish methods to payload
+- feat: auto retry 2 times when "could not obtain a database connection within 5.000 seconds..." error occurs
+
+# 0.5.3 (December 30, 2020)
+- fix: kafka consume all messages from different apps
+- style: use the correct consumer key
+
+# 0.5.2 (December 30, 2020)
+- fix: rabbitmq deliver messages to all subscribers
+- fix: rabbitmq persist messages to recover after restarting
+
+# 0.5.1.1 (December 29, 2020)
+- Hotfix: auto convert class name into string
+
+# 0.5.1 (December 24, 2020)
+- feat: rename publisher callbacks to be more understandable
+- feat: add callbacks to listen when processing a message (before saving sync)
+
 # 0.5.0.1 (December 22, 2020)
 - fix: add missing rabbit mock method
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.0.1)
+    pub_sub_model_sync (0.5.4)
       rails
 
 GEM

data/README.md

@@ -28,10 +28,9 @@ And then execute: $ bundle install
     ```ruby
     # initializers/pub_sub_config.rb
     PubSubModelSync::Config.service_name = :google 
-    PubSubModelSync::Config.project = 'project-id'
+    PubSubModelSync::Config.project = 'google-project-id'
     PubSubModelSync::Config.credentials = 'path-to-the-config'
     PubSubModelSync::Config.topic_name = 'sample-topic'
-    PubSubModelSync::Config.subscription_name = 'p1-subscriber'
     ```
     See details here:
     https://github.com/googleapis/google-cloud-ruby/tree/master/google-cloud-pubsub
@@ -40,7 +39,7 @@ And then execute: $ bundle install
     ```ruby
     PubSubModelSync::Config.service_name = :rabbitmq
     PubSubModelSync::Config.bunny_connection = 'amqp://guest:guest@localhost'
-    PubSubModelSync::Config.queue_name = ''
+    PubSubModelSync::Config.queue_name = 'model-sync'
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/ruby-amqp/bunny
@@ -48,7 +47,7 @@ And then execute: $ bundle install
 - configuration for Apache Kafka (You need kafka installed)
     ```ruby
     PubSubModelSync::Config.service_name = :kafka
-    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], logger: Rails.logger]
+    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], { logger: Rails.logger }]
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/zendesk/ruby-kafka    
@@ -59,7 +58,13 @@ And then execute: $ bundle install
     ```ruby
     rake pub_sub_model_sync:start
     ```
-    Note: Publishers do not need todo this
+    Note: Publishers do not need todo this    
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    ```ruby 
+      # PubSubModelSync::Config.subscribers ==> []
+      Rails.application.try(:eager_load!)
+      # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
+    ``` 
 
 - Check the service status with:    
   ```PubSubModelSync::MessagePublisher.publish_data('Test message', {sample_value: 10}, :create)```
@@ -163,6 +168,9 @@ Note: Be careful with collision of names
   ```.ps_subscriber_changed?(data)```    
   By default: ```model.changed?```
 
+- Permit to perform custom actions before saving sync of the model (:cancel can be returned to skip sync)   
+  ```.ps_before_save_sync(payload)```    
+
 ### Publishers
 - Permit to configure crud publishers
   ```ps_publish(attrs, actions: nil, as_klass: nil)```
@@ -198,10 +206,13 @@ Note: Be careful with collision of names
   * action_name: (required, :sim) Action name    
   * as_klass: (optional, :string) Custom class name (Default current model name)
       
-- Publish a class level notification (Same as above: manual call)    
+- Payload actions  
   ```ruby
     payload = PubSubModelSync::Payload.new({ title: 'hello' }, { action: :greeting, klass: 'User' })
-    payload.publish!
+    payload.publish! # publishes notification data. It raises exception if fails and does not call ```:on_error_publishing``` callback
+    payload.publish # publishes notification data. On error does not raise exception but calls ```:on_error_publishing``` callback
+    payload.process! # process a notification data. It raises exception if fails and does not call ```.on_error_processing``` callback
+    payload.publish # process a notification data. It does not raise exception if fails but calls ```.on_error_processing``` callback
   ```
   
 - Get crud publisher configured for the class   
@@ -274,22 +285,26 @@ config = PubSubModelSync::Config
 config.debug = true
 ```
 
-- ```debug = true```    
+- ```.subscription_name = 'app-2'```    
+    Permit to define a custom consumer identifier (Default: Rails application name)
+- ```.debug = true```    
     (true/false*) => show advanced log messages
-- ```logger = Rails.logger```   
+- ```.logger = Rails.logger```   
     (Logger) => define custom logger
-- ```disabled = true```   
-    (true/false*) => if true, does not publish model messages (Create/Update/Destroy) 
-- ```on_process_success = ->(payload, subscriber) { puts payload }```    
+- ```.disabled_callback_publisher = ->(_model, _action) { false }```   
+    (true/false*) => if true, does not listen model callbacks for auto sync (Create/Update/Destroy) 
+- ```.on_before_processing = ->(payload, {subscriber:}) { puts payload }```    
+    (Proc) => called before processing received message (:cancel can be returned to skip processing)   
+- ```.on_success_processing = ->(payload, {subscriber:}) { puts payload }```    
     (Proc) => called when a message was successfully processed
-- ```on_process_error = ->(exception, payload) { sleep 1; payload.process! }```    
-    (Proc) => called when a message failed when processing
-- ```on_before_publish = ->(payload) { puts payload }```    
-    (Proc) => called before publishing a message    
-- ```on_after_publish = ->(payload) { puts payload }```    
+- ```.on_error_processing = ->(exception, {payload:, subscriber:}) { payload.delay(...).process! }```    
+    (Proc) => called when a message failed when processing (delayed_job or similar can be used for retrying)
+- ```.on_before_publish = ->(payload) { puts payload }```    
+    (Proc) => called before publishing a message (:cancel can be returned to skip publishing)    
+- ```.on_after_publish = ->(payload) { puts payload }```    
     (Proc) => called after publishing a message
-- ```on_publish_error = ->(exception, payload) { sleep 1; payload.publish! }```    
-    (Proc) => called when failed publishing a message
+- ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```    
+    (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)

data/lib/pub_sub_model_sync/base.rb

@@ -13,5 +13,12 @@ module PubSubModelSync
         config.log message, kind
       end
     end
+
+    def retry_error(error_klass, qty: 2, &block)
+      @retries ||= 0
+      block.call
+    rescue error_klass => _e
+      (@retries += 1) <= qty ? retry : raise
+    end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -10,12 +10,13 @@ module PubSubModelSync
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
 
-    cattr_accessor(:on_process_success) { ->(_payload, _subscriber) {} }
-    cattr_accessor(:on_process_error) { ->(_exception, _payload) {} }
-    cattr_accessor(:on_before_publish) { ->(_payload) {} }
+    cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
+    cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
+    cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
+    cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
-    cattr_accessor(:on_publish_error) { ->(_exception, _payload) {} }
-    cattr_accessor(:disabled) { false }
+    cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
+    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
     cattr_accessor :project, :credentials, :topic_name, :subscription_name

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload
+    attr_accessor :payload, :raise_error
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -22,17 +22,27 @@ module PubSubModelSync
     private
 
     def run_subscriber(subscriber)
-      subscriber.eval_message(payload.data)
-      config.on_process_success.call(payload, subscriber)
-      log "processed message with: #{payload}"
+      return unless processable?(subscriber)
+
+      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
+        subscriber.process!(payload)
+        res = config.on_success_processing.call(payload, { subscriber: subscriber })
+        log "processed message with: #{payload.inspect}" if res != :skip_log
+      end
     rescue => e
-      print_subscriber_error(e)
+      raise_error ? raise : print_subscriber_error(e, subscriber)
+    end
+
+    def processable?(subscriber)
+      cancel = config.on_before_processing.call(payload, { subscriber: subscriber }) == :cancel
+      log("process message cancelled: #{payload}") if cancel && config.debug
+      !cancel
     end
 
     # @param error (Error)
-    def print_subscriber_error(error)
+    def print_subscriber_error(error, subscriber)
       info = [payload, error.message, error.backtrace]
-      res = config.on_process_error.call(error, payload)
+      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -8,7 +8,7 @@ module PubSubModelSync
       end
 
       def publish_data(klass, data, action)
-        payload = PubSubModelSync::Payload.new(data, { klass: klass, action: action.to_sym })
+        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
         publish(payload)
       end
 
@@ -19,8 +19,7 @@ module PubSubModelSync
         return if model.ps_skip_sync?(action)
 
         publisher ||= model.class.ps_publisher(action)
-        payload_info = publisher.payload(model, action)
-        payload = PubSubModelSync::Payload.new(payload_info[:data], payload_info[:attrs])
+        payload = publisher.payload(model, action)
         res_before = model.ps_before_sync(action, payload.data)
         return if res_before == :cancel
 
@@ -28,20 +27,24 @@ module PubSubModelSync
         model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload)
-        log("Publishing message: #{[payload]}") if config.debug
-        config.on_before_publish.call(payload)
+      def publish(payload, raise_error: false)
+        if config.on_before_publish.call(payload) == :cancel
+          log("Publish message cancelled: #{payload}") if config.debug
+          return
+        end
+
+        log("Publishing message: #{[payload]}")
         connector.publish(payload)
         config.on_after_publish.call(payload)
       rescue => e
-        notify_error(e, payload)
+        raise_error ? raise : notify_error(e, payload)
       end
 
       private
 
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
-        res = config.on_publish_error.call(exception, payload)
+        res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -26,11 +26,23 @@ module PubSubModelSync
     end
 
     def process!
+      process do |publisher|
+        publisher.raise_error = true
+      end
+    end
+
+    def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
+      yield(publisher) if block_given?
       publisher.process
     end
 
     def publish!
+      klass = PubSubModelSync::MessagePublisher
+      klass.publish(self, raise_error: true)
+    end
+
+    def publish
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self)
     end

data/lib/pub_sub_model_sync/publisher.rb

@@ -12,7 +12,7 @@ module PubSubModelSync
     end
 
     def payload(model, action)
-      { data: payload_data(model), attrs: payload_attrs(model, action) }
+      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
     end
 
     private

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -11,13 +11,12 @@ module PubSubModelSync
       false
     end
 
-    # TODO: make it using respond_to?(:ps_skip_sync?)
     # before preparing data to sync
     def ps_skip_sync?(_action)
       false
     end
 
-    # before delivering data
+    # before delivering data (return :cancel to cancel sync)
     def ps_before_sync(_action, _data); end
 
     # after delivering data
@@ -64,7 +63,7 @@ module PubSubModelSync
 
       def ps_register_callback(action, publisher)
         after_commit(on: action) do |model|
-          disabled = PubSubModelSync::Config.disabled
+          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
           if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
             klass.publish_model(model, action.to_sym, publisher)

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,6 +4,8 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
+    PUBLISH_SETTINGS = {}.freeze
+    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'

data/lib/pub_sub_model_sync/service_google.rb

@@ -18,7 +18,7 @@ module PubSubModelSync
 
     def listen_messages
       @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(&method(:process_message))
+      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
       subscriber.start
       log('Listener started')
@@ -28,7 +28,7 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true })
+      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
     end
 
     def stop
@@ -39,8 +39,8 @@ module PubSubModelSync
     private
 
     def subscribe_to_topic
-      topic.subscription(config.subscription_name) ||
-        topic.subscribe(config.subscription_name)
+      topic.subscription(config.subscription_key) ||
+        topic.subscribe(config.subscription_key)
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -10,17 +10,17 @@ module PubSubModelSync
     cattr_accessor :producer
     attr_accessor :config, :service, :consumer
 
-    CONSUMER_GROUP = 'service_model_sync'
-
     def initialize
       @config = PubSubModelSync::Config
-      @service = Kafka.new(*config.kafka_connection)
+      settings = config.kafka_connection
+      settings[1][:client_id] ||= config.subscription_key
+      @service = Kafka.new(*settings)
     end
 
     def listen_messages
       log('Listener starting...')
       start_consumer
-      consumer.each_message(&method(:process_message))
+      consumer.each_message(LISTEN_SETTINGS, &method(:process_message))
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
     rescue => e
@@ -28,7 +28,11 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      producer.produce(payload.to_json, message_settings)
+      settings = {
+        topic: config.topic_name,
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+      producer.produce(payload.to_json, settings)
       producer.deliver_messages
     end
 
@@ -39,12 +43,8 @@ module PubSubModelSync
 
     private
 
-    def message_settings
-      { topic: config.topic_name, headers: { SERVICE_KEY => true } }
-    end
-
     def start_consumer
-      @consumer = service.consumer(group_id: CONSUMER_GROUP)
+      @consumer = service.consumer(group_id: config.subscription_key)
       consumer.subscribe(config.topic_name)
     end
 

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -40,6 +40,7 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
+      channel&.close
       service.close
     end
 
@@ -48,12 +49,17 @@ module PubSubModelSync
     def message_settings
       {
         routing_key: queue.name,
-        type: SERVICE_KEY
-      }
+        type: SERVICE_KEY,
+        persistent: true
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    def queue_settings
+      { durable: true, auto_delete: false }
     end
 
     def subscribe_settings
-      { manual_ack: false }
+      { manual_ack: false }.merge(LISTEN_SETTINGS)
     end
 
     def process_message(_delivery_info, meta_info, payload)
@@ -65,8 +71,7 @@ module PubSubModelSync
     def subscribe_to_queue
       service.start
       @channel = service.create_channel
-      queue_settings = { durable: true, auto_delete: false }
-      @queue = channel.queue(config.queue_name, queue_settings)
+      @queue = channel.queue(config.subscription_key, queue_settings)
       subscribe_to_exchange
     end
 

data/lib/pub_sub_model_sync/subscriber.rb

@@ -2,7 +2,8 @@
 
 module PubSubModelSync
   class Subscriber
-    attr_accessor :klass, :action, :attrs, :settings
+    attr_accessor :klass, :action, :attrs, :settings, :identifiers
+    attr_reader :payload
 
     # @param settings: (Hash) { id: :id, direct_mode: false,
     #                           from_klass: klass, from_action: action }
@@ -13,50 +14,53 @@ module PubSubModelSync
       @action = action
       @attrs = attrs
       @settings = def_settings.merge(settings)
+      @identifiers = Array(settings[:id]).map(&:to_sym)
     end
 
-    def eval_message(message)
+    def process!(payload)
+      @payload = payload
       if settings[:direct_mode]
-        run_class_message(message)
+        run_class_message
       else
-        run_model_message(message)
+        run_model_message
       end
     end
 
     private
 
-    def run_class_message(message)
+    def run_class_message
       model_class = klass.constantize
-      model_class.send(action, message)
+      model_class.send(action, payload.data)
     end
 
     # support for: create, update, destroy
-    def run_model_message(message)
-      model = find_model(message)
+    def run_model_message
+      model = find_model
+      return if model.ps_before_save_sync(payload) == :cancel
+
       if action == :destroy
         model.destroy!
       else
-        populate_model(model, message)
-        return if action == :update && !model.ps_subscriber_changed?(message)
+        populate_model(model)
+        return if action == :update && !model.ps_subscriber_changed?(payload.data)
 
         model.save!
       end
     end
 
-    def find_model(message)
+    def find_model
       model_class = klass.constantize
-      return model_class.ps_find_model(message) if model_class.respond_to?(:ps_find_model)
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
 
-      model_class.where(model_identifiers(message)).first_or_initialize
+      model_class.where(model_identifiers).first_or_initialize
     end
 
-    def model_identifiers(message)
-      identifiers = Array(settings[:id])
-      identifiers.map { |key| [key, message[key.to_sym]] }.to_h
+    def model_identifiers
+      identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
     end
 
-    def populate_model(model, message)
-      values = message.slice(*attrs)
+    def populate_model(model)
+      values = payload.data.slice(*attrs).except(*identifiers)
       values.each do |attr, value|
         model.send("#{attr}=", value)
       end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -12,6 +12,10 @@ module PubSubModelSync
       changed?
     end
 
+    # permit to apply custom actions before applying sync
+    # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
+    def ps_before_save_sync(_payload); end
+
     module ClassMethods
       def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
         settings = { id: id, from_klass: from_klass }

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.0.1'
+  VERSION = '0.5.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.0.1
+  version: 0.5.4
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails