pub_sub_model_sync 0.5.1.1 → 0.5.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b2f51ef8b439385ed1ff19a342bcfb74571d8d02beb656bc85abb5406e213ee4
-  data.tar.gz: 47cfbad901cd469f91a137f53d2058d9c58e65ff83ee0e2ee9ccb0b2728e398c
+  metadata.gz: 1a19e13bd4fa78cde78749ffa47096832c31abbc3fdc10e3938988c2c4a73d44
+  data.tar.gz: a6abb31c4d900dbb1f7307833e88056ac6d1621bbed46b4136f289602b5aab5c
 SHA512:
-  metadata.gz: 9e5d8da552b0ad819d54aa75a781b4c4848a2c2456d68aa4e60d62291c601107b15c06d5e6c77caff884ed2070493c4291056d532f7674a0a239929bec3abf24
-  data.tar.gz: 6f7f00ac5d1f7bd8be06ae10831446432d7c049ac79339bf704e86af7f1988e775a7a3f158e005f2bdcacaca3a79a82f2f4845aad0e4d5c047d3b465dee89576
+  metadata.gz: 128c4edf32925745e271933c0416c44740c1b553c61f04efb75111787404a419b70185a55527db87d28f4c0d03b534d01ef8e01fa46191b539c1766c315de685
+  data.tar.gz: 4e73d7f16aead95368d4e6830530259e1d664fb0a23b4fc0f5f49e13aead8b5fbc1778aac8c2ea5a0e7c4f996a10f925d3af51c0730bf962ac6d712a11f7bb84

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Change Log
 
+# 0.5.5 (January 11, 2021)
+- feat: google-pub/sub receive messages in the same order they were delivered
+
+# 0.5.4.1 (January 8, 2021)
+- fix: google-pub/sub receive messages sequentially and not in parallel (default 5 threads).
+
+# 0.5.4 (January 8, 2021)
+- fix: exclude identifiers when syncing model
+- feat: callbacks support for future extra params
+- feat: make connectors configurable
+- feat: add :process! and :process, :publish!, :publish methods to payload
+- feat: auto retry 2 times when "could not obtain a database connection within 5.000 seconds..." error occurs
+
+# 0.5.3 (December 30, 2020)
+- fix: kafka consume all messages from different apps
+- style: use the correct consumer key
+
+# 0.5.2 (December 30, 2020)
+- fix: rabbitmq deliver messages to all subscribers
+- fix: rabbitmq persist messages to recover after restarting
+
 # 0.5.1.1 (December 29, 2020)
 - Hotfix: auto convert class name into string
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.1.1)
+    pub_sub_model_sync (0.5.5)
       rails
 
 GEM

data/README.md

@@ -28,10 +28,9 @@ And then execute: $ bundle install
     ```ruby
     # initializers/pub_sub_config.rb
     PubSubModelSync::Config.service_name = :google 
-    PubSubModelSync::Config.project = 'project-id'
+    PubSubModelSync::Config.project = 'google-project-id'
     PubSubModelSync::Config.credentials = 'path-to-the-config'
     PubSubModelSync::Config.topic_name = 'sample-topic'
-    PubSubModelSync::Config.subscription_name = 'p1-subscriber'
     ```
     See details here:
     https://github.com/googleapis/google-cloud-ruby/tree/master/google-cloud-pubsub
@@ -40,7 +39,7 @@ And then execute: $ bundle install
     ```ruby
     PubSubModelSync::Config.service_name = :rabbitmq
     PubSubModelSync::Config.bunny_connection = 'amqp://guest:guest@localhost'
-    PubSubModelSync::Config.queue_name = ''
+    PubSubModelSync::Config.queue_name = 'model-sync'
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/ruby-amqp/bunny
@@ -48,7 +47,7 @@ And then execute: $ bundle install
 - configuration for Apache Kafka (You need kafka installed)
     ```ruby
     PubSubModelSync::Config.service_name = :kafka
-    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], logger: Rails.logger]
+    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], { logger: Rails.logger }]
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/zendesk/ruby-kafka    
@@ -207,10 +206,13 @@ Note: Be careful with collision of names
   * action_name: (required, :sim) Action name    
   * as_klass: (optional, :string) Custom class name (Default current model name)
       
-- Publish a class level notification (Same as above: manual call)    
+- Payload actions  
   ```ruby
     payload = PubSubModelSync::Payload.new({ title: 'hello' }, { action: :greeting, klass: 'User' })
-    payload.publish!
+    payload.publish! # publishes notification data. It raises exception if fails and does not call ```:on_error_publishing``` callback
+    payload.publish # publishes notification data. On error does not raise exception but calls ```:on_error_publishing``` callback
+    payload.process! # process a notification data. It raises exception if fails and does not call ```.on_error_processing``` callback
+    payload.publish # process a notification data. It does not raise exception if fails but calls ```.on_error_processing``` callback
   ```
   
 - Get crud publisher configured for the class   
@@ -283,30 +285,41 @@ config = PubSubModelSync::Config
 config.debug = true
 ```
 
-- ```debug = true```    
+- ```.subscription_name = 'app-2'```    
+    Permit to define a custom consumer identifier (Default: Rails application name)
+- ```.debug = true```    
     (true/false*) => show advanced log messages
-- ```logger = Rails.logger```   
+- ```.logger = Rails.logger```   
     (Logger) => define custom logger
-- ```disabled_callback_publisher = ->(_model, _action) { false }```   
+- ```.disabled_callback_publisher = ->(_model, _action) { false }```   
     (true/false*) => if true, does not listen model callbacks for auto sync (Create/Update/Destroy) 
-- ```on_before_processing = ->(payload, subscriber) { puts payload }```    
+- ```.on_before_processing = ->(payload, {subscriber:}) { puts payload }```    
     (Proc) => called before processing received message (:cancel can be returned to skip processing)   
-- ```on_success_processing = ->(payload, subscriber) { puts payload }```    
+- ```.on_success_processing = ->(payload, {subscriber:}) { puts payload }```    
     (Proc) => called when a message was successfully processed
-- ```on_error_processing = ->(exception, payload) { sleep 1; payload.process! }```    
-    (Proc) => called when a message failed when processing
-- ```on_before_publish = ->(payload) { puts payload }```    
+- ```.on_error_processing = ->(exception, {payload:, subscriber:}) { payload.delay(...).process! }```    
+    (Proc) => called when a message failed when processing (delayed_job or similar can be used for retrying)
+- ```.on_before_publish = ->(payload) { puts payload }```    
     (Proc) => called before publishing a message (:cancel can be returned to skip publishing)    
-- ```on_after_publish = ->(payload) { puts payload }```    
+- ```.on_after_publish = ->(payload) { puts payload }```    
     (Proc) => called after publishing a message
-- ```on_error_publish = ->(exception, payload) { sleep 1; payload.publish! }```    
-    (Proc) => called when failed publishing a message
+- ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```    
+    (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)
 - Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
 - Auto publish update only if payload has changed
-- On delete, payload must only be composed by ids 
+- On delete, payload must only be composed by ids
+- Change notifications into messages 
+
+## Q&A
+- Error "could not obtain a connection from the pool within 5.000 seconds"    
+  This problem occurs because pub/sub dependencies (kafka, google-pubsub, rabbitmq) use many threads to perform notifications where the qty of threads is greater than qty of DB pools ([Google pubsub info](https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-pubsub/lib/google/cloud/pubsub/subscription.rb#L888))     
+  To fix the problem, edit config/database.yml and increase the quantity of ```pool: 10```
+- Google pubsub: How to process notifications parallely and not sequentially (default 1 thread)?    
+  ```ruby  PubSubModelSync::ServiceGoogle::LISTEN_SETTINGS = { threads: { callback: qty_threads } } ```    
+  Note: by this way some notifications can be processed before others thus missing relationship errors can appear 
 
 ## Contributing
 

data/lib/pub_sub_model_sync/base.rb

@@ -13,5 +13,12 @@ module PubSubModelSync
         config.log message, kind
       end
     end
+
+    def retry_error(error_klass, qty: 2, &block)
+      @retries ||= 0
+      block.call
+    rescue error_klass => _e
+      (@retries += 1) <= qty ? retry : raise
+    end
   end
 end

data/lib/pub_sub_model_sync/config.rb

@@ -10,12 +10,12 @@ module PubSubModelSync
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
 
-    cattr_accessor(:on_before_processing) { ->(_payload, _subscriber) {} } # return :cancel to skip
-    cattr_accessor(:on_success_processing) { ->(_payload, _subscriber) {} }
-    cattr_accessor(:on_error_processing) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
+    cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
+    cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
-    cattr_accessor(:on_error_publish) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }
     cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload
+    attr_accessor :payload, :raise_error
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -24,23 +24,25 @@ module PubSubModelSync
     def run_subscriber(subscriber)
       return unless processable?(subscriber)
 
-      subscriber.process!(payload)
-      config.on_success_processing.call(payload, subscriber)
-      log "processed message with: #{payload}"
+      retry_error(ActiveRecord::ConnectionTimeoutError, qty: 2) do
+        subscriber.process!(payload)
+        res = config.on_success_processing.call(payload, { subscriber: subscriber })
+        log "processed message with: #{payload.inspect}" if res != :skip_log
+      end
     rescue => e
-      print_subscriber_error(e)
+      raise_error ? raise : print_subscriber_error(e, subscriber)
     end
 
     def processable?(subscriber)
-      cancel = config.on_before_processing.call(payload, subscriber) == :cancel
+      cancel = config.on_before_processing.call(payload, { subscriber: subscriber }) == :cancel
       log("process message cancelled: #{payload}") if cancel && config.debug
       !cancel
     end
 
     # @param error (Error)
-    def print_subscriber_error(error)
+    def print_subscriber_error(error, subscriber)
       info = [payload, error.message, error.backtrace]
-      res = config.on_error_processing.call(error, payload)
+      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -19,8 +19,7 @@ module PubSubModelSync
         return if model.ps_skip_sync?(action)
 
         publisher ||= model.class.ps_publisher(action)
-        payload_info = publisher.payload(model, action)
-        payload = PubSubModelSync::Payload.new(payload_info[:data], payload_info[:attrs])
+        payload = publisher.payload(model, action)
         res_before = model.ps_before_sync(action, payload.data)
         return if res_before == :cancel
 
@@ -28,7 +27,7 @@ module PubSubModelSync
         model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload)
+      def publish(payload, raise_error: false)
         if config.on_before_publish.call(payload) == :cancel
           log("Publish message cancelled: #{payload}") if config.debug
           return
@@ -38,14 +37,14 @@ module PubSubModelSync
         connector.publish(payload)
         config.on_after_publish.call(payload)
       rescue => e
-        notify_error(e, payload)
+        raise_error ? raise : notify_error(e, payload)
       end
 
       private
 
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
-        res = config.on_error_publish.call(exception, payload)
+        res = config.on_error_publish.call(exception, { payload: payload })
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end

data/lib/pub_sub_model_sync/payload.rb

@@ -26,11 +26,23 @@ module PubSubModelSync
     end
 
     def process!
+      process do |publisher|
+        publisher.raise_error = true
+      end
+    end
+
+    def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
+      yield(publisher) if block_given?
       publisher.process
     end
 
     def publish!
+      klass = PubSubModelSync::MessagePublisher
+      klass.publish(self, raise_error: true)
+    end
+
+    def publish
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self)
     end

data/lib/pub_sub_model_sync/publisher.rb

@@ -12,7 +12,7 @@ module PubSubModelSync
     end
 
     def payload(model, action)
-      { data: payload_data(model), attrs: payload_attrs(model, action) }
+      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
     end
 
     private

data/lib/pub_sub_model_sync/service_base.rb

@@ -4,6 +4,8 @@ require 'pub_sub_model_sync/payload'
 module PubSubModelSync
   class ServiceBase < PubSubModelSync::Base
     SERVICE_KEY = 'service_model_sync'
+    PUBLISH_SETTINGS = {}.freeze
+    LISTEN_SETTINGS = {}.freeze
 
     def listen_messages
       raise 'method :listen_messages must be defined in service'

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,6 +7,8 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
+    LISTEN_SETTINGS = { threads: { callback: 1 } }.freeze
+    SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
     attr_accessor :service, :topic, :subscription, :subscriber
 
     def initialize
@@ -18,7 +20,7 @@ module PubSubModelSync
 
     def listen_messages
       @subscription = subscribe_to_topic
-      @subscriber = subscription.listen(&method(:process_message))
+      @subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
       log('Listener starting...')
       subscriber.start
       log('Listener started')
@@ -28,7 +30,7 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true })
+      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
     end
 
     def stop
@@ -39,8 +41,8 @@ module PubSubModelSync
     private
 
     def subscribe_to_topic
-      topic.subscription(config.subscription_name) ||
-        topic.subscribe(config.subscription_name)
+      topic.subscription(config.subscription_key) ||
+        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -10,17 +10,17 @@ module PubSubModelSync
     cattr_accessor :producer
     attr_accessor :config, :service, :consumer
 
-    CONSUMER_GROUP = 'service_model_sync'
-
     def initialize
       @config = PubSubModelSync::Config
-      @service = Kafka.new(*config.kafka_connection)
+      settings = config.kafka_connection
+      settings[1][:client_id] ||= config.subscription_key
+      @service = Kafka.new(*settings)
     end
 
     def listen_messages
       log('Listener starting...')
       start_consumer
-      consumer.each_message(&method(:process_message))
+      consumer.each_message(LISTEN_SETTINGS, &method(:process_message))
     rescue PubSubModelSync::Runner::ShutDown
       log('Listener stopped')
     rescue => e
@@ -28,7 +28,11 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      producer.produce(payload.to_json, message_settings)
+      settings = {
+        topic: config.topic_name,
+        headers: { SERVICE_KEY => true }
+      }.merge(PUBLISH_SETTINGS)
+      producer.produce(payload.to_json, settings)
       producer.deliver_messages
     end
 
@@ -39,12 +43,8 @@ module PubSubModelSync
 
     private
 
-    def message_settings
-      { topic: config.topic_name, headers: { SERVICE_KEY => true } }
-    end
-
     def start_consumer
-      @consumer = service.consumer(group_id: CONSUMER_GROUP)
+      @consumer = service.consumer(group_id: config.subscription_key)
       consumer.subscribe(config.topic_name)
     end
 

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -40,6 +40,7 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
+      channel&.close
       service.close
     end
 
@@ -48,12 +49,17 @@ module PubSubModelSync
     def message_settings
       {
         routing_key: queue.name,
-        type: SERVICE_KEY
-      }
+        type: SERVICE_KEY,
+        persistent: true
+      }.merge(PUBLISH_SETTINGS)
+    end
+
+    def queue_settings
+      { durable: true, auto_delete: false }
     end
 
     def subscribe_settings
-      { manual_ack: false }
+      { manual_ack: false }.merge(LISTEN_SETTINGS)
     end
 
     def process_message(_delivery_info, meta_info, payload)
@@ -65,8 +71,7 @@ module PubSubModelSync
     def subscribe_to_queue
       service.start
       @channel = service.create_channel
-      queue_settings = { durable: true, auto_delete: false }
-      @queue = channel.queue(config.queue_name, queue_settings)
+      @queue = channel.queue(config.subscription_key, queue_settings)
       subscribe_to_exchange
     end
 

data/lib/pub_sub_model_sync/subscriber.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class Subscriber
-    attr_accessor :klass, :action, :attrs, :settings
+    attr_accessor :klass, :action, :attrs, :settings, :identifiers
     attr_reader :payload
 
     # @param settings: (Hash) { id: :id, direct_mode: false,
@@ -14,6 +14,7 @@ module PubSubModelSync
       @action = action
       @attrs = attrs
       @settings = def_settings.merge(settings)
+      @identifiers = Array(settings[:id]).map(&:to_sym)
     end
 
     def process!(payload)
@@ -55,12 +56,11 @@ module PubSubModelSync
     end
 
     def model_identifiers
-      identifiers = Array(settings[:id])
       identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
     end
 
     def populate_model(model)
-      values = payload.data.slice(*attrs)
+      values = payload.data.slice(*attrs).except(*identifiers)
       values.each do |attr, value|
         model.send("#{attr}=", value)
       end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.1.1'
+  VERSION = '0.5.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.1.1
+  version: 0.5.5
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-29 00:00:00.000000000 Z
+date: 2021-01-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails