pub_sub_model_sync 0.5.3 → 0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 608c6430e2e37529a6a219d09db34b7a540d89bed73da7e272c1624ad7ba821f
-  data.tar.gz: 99eaf893d6c66d3e7429d6ba8758b7e44d9ca0a29069b72f260bccecd7a207f0
+  metadata.gz: 57e8d23c9e514734a8a45664f641378bbb46e8de5315c536a0f692efcbe29dd2
+  data.tar.gz: d0f290dac2b0563ffe5412f85e1a6a6e340e281049755b26dc732a3661501b92
 SHA512:
-  metadata.gz: c9a0d769a2c18b9097d7f9d4b10f8f11567e743a39bcf10fe39ed4bd3daf156fb028685be4f8100d92d9bb0fd8ea742a1c0a5dca8fe0743c0c4e21b3b23fd552
-  data.tar.gz: 4ff19cc02656fd8ddae04e061808c89ec61eae3e86cd889a3c2abc3fed0c2975ed8eed995363ad834d294f75ac340fdeb454c1de21e05e2f4080458664f8d181
+  metadata.gz: 351901f98e6c806e7a4efe20abdac65de5bf8f716dc57e1889302eab02d33b3f2f4d449cd0d351fd5ae882d16abe79a77791bea8ad956c52cda47829319eb858
+  data.tar.gz: e906d601a8c37b999a8b6d550e605e9c04b67db692cec4b60220ae286bef9adde7132d8614cee4ca7dab9ef43538ae2baede3f07e402c4c21a189b79a9ba6873

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Change Log
 
+# 0.5.4 (January 8, 2021)
+- fix: exclude identifiers when syncing model
+- feat: callbacks support for future extra params
+- feat: make connectors configurable
+- feat: add :process! and :process, :publish!, :publish methods to payload
+- feat: auto retry 2 times when "could not obtain a database connection within 5.000 seconds..." error occurs
+
 # 0.5.3 (December 30, 2020)
 - fix: kafka consume all messages from different apps
 - style: use the correct consumer key

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.3)
+    pub_sub_model_sync (0.5.4)
       rails
 
 GEM

data/README.md

@@ -206,10 +206,13 @@ Note: Be careful with collision of names
   * action_name: (required, :sim) Action name    
   * as_klass: (optional, :string) Custom class name (Default current model name)
       
-- Publish a class level notification (Same as above: manual call)    
+- Payload actions  
   ```ruby
     payload = PubSubModelSync::Payload.new({ title: 'hello' }, { action: :greeting, klass: 'User' })
-    payload.publish!
+    payload.publish! # publishes notification data. It raises exception if fails and does not call ```:on_error_publishing``` callback
+    payload.publish # publishes notification data. On error does not raise exception but calls ```:on_error_publishing``` callback
+    payload.process! # process a notification data. It raises exception if fails and does not call ```.on_error_processing``` callback
+    payload.publish # process a notification data. It does not raise exception if fails but calls ```.on_error_processing``` callback
   ```
   
 - Get crud publisher configured for the class   
@@ -290,18 +293,18 @@ config.debug = true
     (Logger) => define custom logger
 - ```.disabled_callback_publisher = ->(_model, _action) { false }```   
     (true/false*) => if true, does not listen model callbacks for auto sync (Create/Update/Destroy) 
-- ```.on_before_processing = ->(payload, subscriber) { puts payload }```    
+- ```.on_before_processing = ->(payload, {subscriber:}) { puts payload }```    
     (Proc) => called before processing received message (:cancel can be returned to skip processing)   
-- ```.on_success_processing = ->(payload, subscriber) { puts payload }```    
+- ```.on_success_processing = ->(payload, {subscriber:}) { puts payload }```    
     (Proc) => called when a message was successfully processed
-- ```.on_error_processing = ->(exception, payload) { sleep 1; payload.process! }```    
-    (Proc) => called when a message failed when processing
+- ```.on_error_processing = ->(exception, {payload:, subscriber:}) { payload.delay(...).process! }```    
+    (Proc) => called when a message failed when processing (delayed_job or similar can be used for retrying)
 - ```.on_before_publish = ->(payload) { puts payload }```    
     (Proc) => called before publishing a message (:cancel can be returned to skip publishing)    
 - ```.on_after_publish = ->(payload) { puts payload }```    
     (Proc) => called after publishing a message
-- ```.on_error_publish = ->(exception, payload) { sleep 1; payload.publish! }```    
-    (Proc) => called when failed publishing a message
+- ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```    
+    (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)

data/lib/pub_sub_model_sync/base.rb

@@ -13,5 +13,12 @@ module PubSubModelSync
         config.log message, kind
       end
     end
+
+    def retry_error(error_klass, qty: 2, &block)
+      @retries ||= 0
+      block.call
+    rescue error_klass => _e
+      (@retries += 1) <= qty ? retry : raise
+    end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -10,12 +10,12 @@ module PubSubModelSync
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
 
-    cattr_accessor(:on_before_processing) { ->(_payload, _subscriber) {} } # return :cancel to skip
-    cattr_accessor(:on_success_processing) { ->(_payload, _subscriber) {} }
-    cattr_accessor(:on_error_processing) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
+    cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
+    cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
-    cattr_accessor(:on_error_publish) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
     cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload
+    attr_accessor :payload, :raise_error
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -24,23 +24,25 @@ module PubSubModelSync
     def run_subscriber(subscriber)
       return unless processable?(subscriber)
 
-      subscriber.process!(payload)
-      res = config.on_success_processing.call(payload, subscriber)
-      log "processed message with: #{payload.inspect}" if res != :skip_log
+      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
+        subscriber.process!(payload)
+        res = config.on_success_processing.call(payload, { subscriber: subscriber })
+        log "processed message with: #{payload.inspect}" if res != :skip_log
+      end
     rescue => e
-      print_subscriber_error(e)
+      raise_error ? raise : print_subscriber_error(e, subscriber)
     end
 
     def processable?(subscriber)
-      cancel = config.on_before_processing.call(payload, subscriber) == :cancel
+      cancel = config.on_before_processing.call(payload, { subscriber: subscriber }) == :cancel
       log("process message cancelled: #{payload}") if cancel && config.debug
       !cancel
     end
 
     # @param error (Error)
-    def print_subscriber_error(error)
+    def print_subscriber_error(error, subscriber)
       info = [payload, error.message, error.backtrace]
-      res = config.on_error_processing.call(error, payload)
+      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -27,7 +27,7 @@ module PubSubModelSync
         model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload)
+      def publish(payload, raise_error: false)
         if config.on_before_publish.call(payload) == :cancel
           log("Publish message cancelled: #{payload}") if config.debug
           return
@@ -37,14 +37,14 @@ module PubSubModelSync
         connector.publish(payload)
         config.on_after_publish.call(payload)
       rescue => e
-        notify_error(e, payload)
+        raise_error ? raise : notify_error(e, payload)
       end
 
       private
 
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
-        res = config.on_error_publish.call(exception, payload)
+        res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -26,11 +26,23 @@ module PubSubModelSync
     end
 
     def process!
+      process do |publisher|
+        publisher.raise_error = true
+      end
+    end
+
+    def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
+      yield(publisher) if block_given?
       publisher.process
     end
 
     def publish!
+      klass = PubSubModelSync::MessagePublisher
+      klass.publish(self, raise_error: true)
+    end
+
+    def publish
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self)
     end

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,6 +4,8 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
+    PUBLISH_SETTINGS = {}.freeze
+    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'

data/lib/pub_sub_model_sync/service_google.rb

@@ -18,7 +18,7 @@ module PubSubModelSync
 
     def listen_messages
       @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(&method(:process_message))
+      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
       subscriber.start
       log('Listener started')
@@ -28,7 +28,7 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true })
+      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
     end
 
     def stop

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -20,7 +20,7 @@ module PubSubModelSync
     def listen_messages
       log('Listener starting...')
       start_consumer
-      consumer.each_message(&method(:process_message))
+      consumer.each_message(LISTEN_SETTINGS, &method(:process_message))
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
     rescue => e
@@ -28,7 +28,11 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      producer.produce(payload.to_json, message_settings)
+      settings = {
+        topic: config.topic_name,
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+      producer.produce(payload.to_json, settings)
       producer.deliver_messages
     end
 
@@ -39,10 +43,6 @@ module PubSubModelSync
 
     private
 
-    def message_settings
-      { topic: config.topic_name, headers: { SERVICE_KEY => true } }
-    end
-
     def start_consumer
       @consumer = service.consumer(group_id: config.subscription_key)
       consumer.subscribe(config.topic_name)

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -51,7 +51,7 @@ module PubSubModelSync
         routing_key: queue.name,
         type: SERVICE_KEY,
         persistent: true
-      }
+      }.merge(PUBLISH_SETTINGS)
     end
 
     def queue_settings
@@ -59,7 +59,7 @@ module PubSubModelSync
     end
 
     def subscribe_settings
-      { manual_ack: false }
+      { manual_ack: false }.merge(LISTEN_SETTINGS)
     end
 
     def process_message(_delivery_info, meta_info, payload)

data/lib/pub_sub_model_sync/subscriber.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Subscriber
-    attr_accessor :klass, :action, :attrs, :settings
+    attr_accessor :klass, :action, :attrs, :settings, :identifiers
     attr_reader :payload
 
     # @param settings: (Hash) { id: :id, direct_mode: false,
@@ -14,6 +14,7 @@ module PubSubModelSync
       @action = action
       @attrs = attrs
       @settings = def_settings.merge(settings)
+      @identifiers = Array(settings[:id]).map(&:to_sym)
     end
 
     def process!(payload)
@@ -55,12 +56,11 @@ module PubSubModelSync
     end
 
     def model_identifiers
-      identifiers = Array(settings[:id])
       identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
     end
 
     def populate_model(model)
-      values = payload.data.slice(*attrs)
+      values = payload.data.slice(*attrs).except(*identifiers)
       values.each do |attr, value|
         model.send("#{attr}=", value)
       end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.3'
+  VERSION = '0.5.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.3
+  version: 0.5.4
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-31 00:00:00.000000000 Z
+date: 2021-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails