pub_sub_model_sync 0.5.0 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: edcb2024203ae281bbc642825d0f5361205afe787c4a2ebb735cefaa1d98b80c
-  data.tar.gz: 1d6bb21f6808f9595d0bb842a997457097bb0dbc25deb500bb67d73f229b2480
+  metadata.gz: 608c6430e2e37529a6a219d09db34b7a540d89bed73da7e272c1624ad7ba821f
+  data.tar.gz: 99eaf893d6c66d3e7429d6ba8758b7e44d9ca0a29069b72f260bccecd7a207f0
 SHA512:
-  metadata.gz: c6fd7fd582186a12564a41314ba85b198011213b79e6c18af24a542652d30b2ab2f44126f501608a37cdd7e1714cba84cb174b1ae63a14086d0a691acee1a4d4
-  data.tar.gz: 6ed6f1c39199582db6e7b51468e4da9ff98b55c8245df64971bc76bf510bf2f68e15109104074fbb5a3e34a1534c1711990f514b12b22d1e6504560c4a19612f
+  metadata.gz: c9a0d769a2c18b9097d7f9d4b10f8f11567e743a39bcf10fe39ed4bd3daf156fb028685be4f8100d92d9bb0fd8ea742a1c0a5dca8fe0743c0c4e21b3b23fd552
+  data.tar.gz: 4ff19cc02656fd8ddae04e061808c89ec61eae3e86cd889a3c2abc3fed0c2975ed8eed995363ad834d294f75ac340fdeb454c1de21e05e2f4080458664f8d181

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Change Log
 
+# 0.5.3 (December 30, 2020)
+- fix: kafka consume all messages from different apps
+- style: use the correct consumer key
+
+# 0.5.2 (December 30, 2020)
+- fix: rabbitmq deliver messages to all subscribers
+- fix: rabbitmq persist messages to recover after restarting
+
+# 0.5.1.1 (December 29, 2020)
+- Hotfix: auto convert class name into string
+
+# 0.5.1 (December 24, 2020)
+- feat: rename publisher callbacks to be more understandable
+- feat: add callbacks to listen when processing a message (before saving sync)
+
+# 0.5.0.1 (December 22, 2020)
+- fix: add missing rabbit mock method
+
 # 0.5.0 (December 22, 2020)
 - feat: add :publish! and :process! methods to payloads
 - feat: add ability to disable publisher globally

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.0)
+    pub_sub_model_sync (0.5.3)
       rails
 
 GEM
@@ -77,7 +77,7 @@ GEM
       database_cleaner (~> 1.8.0)
     diff-lcs (1.3)
     digest-crc (0.5.1)
-    erubi (1.9.0)
+    erubi (1.10.0)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
     globalid (0.4.2)
@@ -121,7 +121,7 @@ GEM
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
     jwt (2.2.1)
-    loofah (2.6.0)
+    loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -136,7 +136,7 @@ GEM
     minitest (5.14.0)
     multi_json (1.14.1)
     multipart-post (2.1.1)
-    nio4r (2.5.2)
+    nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
     os (1.0.1)
@@ -213,7 +213,7 @@ GEM
     sprockets (4.0.2)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.1)
+    sprockets-rails (3.2.2)
       actionpack (>= 4.0)
       activesupport (>= 4.0)
       sprockets (>= 3.0.0)

data/README.md

@@ -28,10 +28,9 @@ And then execute: $ bundle install
     ```ruby
     # initializers/pub_sub_config.rb
     PubSubModelSync::Config.service_name = :google 
-    PubSubModelSync::Config.project = 'project-id'
+    PubSubModelSync::Config.project = 'google-project-id'
     PubSubModelSync::Config.credentials = 'path-to-the-config'
     PubSubModelSync::Config.topic_name = 'sample-topic'
-    PubSubModelSync::Config.subscription_name = 'p1-subscriber'
     ```
     See details here:
     https://github.com/googleapis/google-cloud-ruby/tree/master/google-cloud-pubsub
@@ -40,7 +39,7 @@ And then execute: $ bundle install
     ```ruby
     PubSubModelSync::Config.service_name = :rabbitmq
     PubSubModelSync::Config.bunny_connection = 'amqp://guest:guest@localhost'
-    PubSubModelSync::Config.queue_name = ''
+    PubSubModelSync::Config.queue_name = 'model-sync'
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/ruby-amqp/bunny
@@ -48,7 +47,7 @@ And then execute: $ bundle install
 - configuration for Apache Kafka (You need kafka installed)
     ```ruby
     PubSubModelSync::Config.service_name = :kafka
-    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], logger: Rails.logger]
+    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], { logger: Rails.logger }]
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/zendesk/ruby-kafka    
@@ -59,7 +58,13 @@ And then execute: $ bundle install
     ```ruby
     rake pub_sub_model_sync:start
     ```
-    Note: Publishers do not need todo this
+    Note: Publishers do not need todo this    
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    ```ruby 
+      # PubSubModelSync::Config.subscribers ==> []
+      Rails.application.try(:eager_load!)
+      # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
+    ``` 
 
 - Check the service status with:    
   ```PubSubModelSync::MessagePublisher.publish_data('Test message', {sample_value: 10}, :create)```
@@ -163,6 +168,9 @@ Note: Be careful with collision of names
   ```.ps_subscriber_changed?(data)```    
   By default: ```model.changed?```
 
+- Permit to perform custom actions before saving sync of the model (:cancel can be returned to skip sync)   
+  ```.ps_before_save_sync(payload)```    
+
 ### Publishers
 - Permit to configure crud publishers
   ```ps_publish(attrs, actions: nil, as_klass: nil)```
@@ -274,21 +282,25 @@ config = PubSubModelSync::Config
 config.debug = true
 ```
 
-- ```debug = true```    
+- ```.subscription_name = 'app-2'```    
+    Permit to define a custom consumer identifier (Default: Rails application name)
+- ```.debug = true```    
     (true/false*) => show advanced log messages
-- ```logger = Rails.logger```   
+- ```.logger = Rails.logger```   
     (Logger) => define custom logger
-- ```disabled = true```   
-    (true/false*) => if true, does not publish model messages (Create/Update/Destroy) 
-- ```on_process_success = ->(payload, subscriber) { puts payload }```    
+- ```.disabled_callback_publisher = ->(_model, _action) { false }```   
+    (true/false*) => if true, does not listen model callbacks for auto sync (Create/Update/Destroy) 
+- ```.on_before_processing = ->(payload, subscriber) { puts payload }```    
+    (Proc) => called before processing received message (:cancel can be returned to skip processing)   
+- ```.on_success_processing = ->(payload, subscriber) { puts payload }```    
     (Proc) => called when a message was successfully processed
-- ```on_process_error = ->(exception, payload) { sleep 1; payload.process! }```    
+- ```.on_error_processing = ->(exception, payload) { sleep 1; payload.process! }```    
     (Proc) => called when a message failed when processing
-- ```on_before_publish = ->(payload) { puts payload }```    
-    (Proc) => called before publishing a message    
-- ```on_after_publish = ->(payload) { puts payload }```    
+- ```.on_before_publish = ->(payload) { puts payload }```    
+    (Proc) => called before publishing a message (:cancel can be returned to skip publishing)    
+- ```.on_after_publish = ->(payload) { puts payload }```    
     (Proc) => called after publishing a message
-- ```on_publish_error = ->(exception, payload) { sleep 1; payload.publish! }```    
+- ```.on_error_publish = ->(exception, payload) { sleep 1; payload.publish! }```    
     (Proc) => called when failed publishing a message
     
 ## TODO

data/lib/pub_sub_model_sync/config.rb

@@ -10,12 +10,13 @@ module PubSubModelSync
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
 
-    cattr_accessor(:on_process_success) { ->(_payload, _subscriber) {} }
-    cattr_accessor(:on_process_error) { ->(_exception, _payload) {} }
-    cattr_accessor(:on_before_publish) { ->(_payload) {} }
+    cattr_accessor(:on_before_processing) { ->(_payload, _subscriber) {} } # return :cancel to skip
+    cattr_accessor(:on_success_processing) { ->(_payload, _subscriber) {} }
+    cattr_accessor(:on_error_processing) { ->(_exception, _payload) {} }
+    cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
-    cattr_accessor(:on_publish_error) { ->(_exception, _payload) {} }
-    cattr_accessor(:disabled) { false }
+    cattr_accessor(:on_error_publish) { ->(_exception, _payload) {} }
+    cattr_accessor(:disabled_callback_publisher) { ->(_model, _action) { false } }
 
     # google service
     cattr_accessor :project, :credentials, :topic_name, :subscription_name

data/lib/pub_sub_model_sync/message_processor.rb

@@ -22,17 +22,25 @@ module PubSubModelSync
     private
 
     def run_subscriber(subscriber)
-      subscriber.eval_message(payload.data)
-      config.on_process_success.call(payload, subscriber)
-      log "processed message with: #{payload}"
+      return unless processable?(subscriber)
+
+      subscriber.process!(payload)
+      res = config.on_success_processing.call(payload, subscriber)
+      log "processed message with: #{payload.inspect}" if res != :skip_log
     rescue => e
       print_subscriber_error(e)
     end
 
+    def processable?(subscriber)
+      cancel = config.on_before_processing.call(payload, subscriber) == :cancel
+      log("process message cancelled: #{payload}") if cancel && config.debug
+      !cancel
+    end
+
     # @param error (Error)
     def print_subscriber_error(error)
       info = [payload, error.message, error.backtrace]
-      res = config.on_process_error.call(error, payload)
+      res = config.on_error_processing.call(error, payload)
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -8,7 +8,7 @@ module PubSubModelSync
       end
 
       def publish_data(klass, data, action)
-        payload = PubSubModelSync::Payload.new(data, { klass: klass, action: action.to_sym })
+        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
         publish(payload)
       end
 
@@ -19,8 +19,7 @@ module PubSubModelSync
         return if model.ps_skip_sync?(action)
 
         publisher ||= model.class.ps_publisher(action)
-        payload_info = publisher.payload(model, action)
-        payload = PubSubModelSync::Payload.new(payload_info[:data], payload_info[:attrs])
+        payload = publisher.payload(model, action)
         res_before = model.ps_before_sync(action, payload.data)
         return if res_before == :cancel
 
@@ -29,8 +28,12 @@ module PubSubModelSync
       end
 
       def publish(payload)
-        log("Publishing message: #{[payload]}") if config.debug
-        config.on_before_publish.call(payload)
+        if config.on_before_publish.call(payload) == :cancel
+          log("Publish message cancelled: #{payload}") if config.debug
+          return
+        end
+
+        log("Publishing message: #{[payload]}")
         connector.publish(payload)
         config.on_after_publish.call(payload)
       rescue => e
@@ -41,7 +44,7 @@ module PubSubModelSync
 
       def notify_error(exception, payload)
         info = [payload, exception.message, exception.backtrace]
-        res = config.on_publish_error.call(exception, payload)
+        res = config.on_error_publish.call(exception, payload)
         log("Error publishing: #{info}", :error) if res != :skip_log
       end
     end

data/lib/pub_sub_model_sync/mock_rabbit_service.rb

@@ -20,6 +20,10 @@ module PubSubModelSync
       def name
         'name'
       end
+
+      def publish(*_args)
+        true
+      end
     end
 
     class MockChannel

data/lib/pub_sub_model_sync/publisher.rb

@@ -12,7 +12,7 @@ module PubSubModelSync
     end
 
     def payload(model, action)
-      { data: payload_data(model), attrs: payload_attrs(model, action) }
+      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
     end
 
     private

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -11,13 +11,12 @@ module PubSubModelSync
       false
     end
 
-    # TODO: make it using respond_to?(:ps_skip_sync?)
     # before preparing data to sync
     def ps_skip_sync?(_action)
       false
     end
 
-    # before delivering data
+    # before delivering data (return :cancel to cancel sync)
     def ps_before_sync(_action, _data); end
 
     # after delivering data
@@ -64,7 +63,7 @@ module PubSubModelSync
 
       def ps_register_callback(action, publisher)
         after_commit(on: action) do |model|
-          disabled = PubSubModelSync::Config.disabled
+          disabled = PubSubModelSync::Config.disabled_callback_publisher.call(model, action)
           if !disabled && !model.ps_skip_callback?(action)
             klass = PubSubModelSync::MessagePublisher
             klass.publish_model(model, action.to_sym, publisher)

data/lib/pub_sub_model_sync/service_google.rb

@@ -39,8 +39,8 @@ module PubSubModelSync
     private
 
     def subscribe_to_topic
-      topic.subscription(config.subscription_name) ||
-        topic.subscribe(config.subscription_name)
+      topic.subscription(config.subscription_key) ||
+        topic.subscribe(config.subscription_key)
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -10,11 +10,11 @@ module PubSubModelSync
     cattr_accessor :producer
     attr_accessor :config, :service, :consumer
 
-    CONSUMER_GROUP = 'service_model_sync'
-
     def initialize
       @config = PubSubModelSync::Config
-      @service = Kafka.new(*config.kafka_connection)
+      settings = config.kafka_connection
+      settings[1][:client_id] ||= config.subscription_key
+      @service = Kafka.new(*settings)
     end
 
     def listen_messages
@@ -44,7 +44,7 @@ module PubSubModelSync
     end
 
     def start_consumer
-      @consumer = service.consumer(group_id: CONSUMER_GROUP)
+      @consumer = service.consumer(group_id: config.subscription_key)
       consumer.subscribe(config.topic_name)
     end
 

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -40,6 +40,7 @@ module PubSubModelSync
 
     def stop
       log('Listener stopping...')
+      channel&.close
       service.close
     end
 
@@ -48,10 +49,15 @@ module PubSubModelSync
     def message_settings
       {
         routing_key: queue.name,
-        type: SERVICE_KEY
+        type: SERVICE_KEY,
+        persistent: true
       }
     end
 
+    def queue_settings
+      { durable: true, auto_delete: false }
+    end
+
     def subscribe_settings
       { manual_ack: false }
     end
@@ -65,8 +71,7 @@ module PubSubModelSync
     def subscribe_to_queue
       service.start
       @channel = service.create_channel
-      queue_settings = { durable: true, auto_delete: false }
-      @queue = channel.queue(config.queue_name, queue_settings)
+      @queue = channel.queue(config.subscription_key, queue_settings)
       subscribe_to_exchange
     end
 

data/lib/pub_sub_model_sync/subscriber.rb

@@ -3,6 +3,7 @@
 module PubSubModelSync
   class Subscriber
     attr_accessor :klass, :action, :attrs, :settings
+    attr_reader :payload
 
     # @param settings: (Hash) { id: :id, direct_mode: false,
     #                           from_klass: klass, from_action: action }
@@ -15,48 +16,51 @@ module PubSubModelSync
       @settings = def_settings.merge(settings)
     end
 
-    def eval_message(message)
+    def process!(payload)
+      @payload = payload
       if settings[:direct_mode]
-        run_class_message(message)
+        run_class_message
       else
-        run_model_message(message)
+        run_model_message
       end
     end
 
     private
 
-    def run_class_message(message)
+    def run_class_message
       model_class = klass.constantize
-      model_class.send(action, message)
+      model_class.send(action, payload.data)
     end
 
     # support for: create, update, destroy
-    def run_model_message(message)
-      model = find_model(message)
+    def run_model_message
+      model = find_model
+      return if model.ps_before_save_sync(payload) == :cancel
+
       if action == :destroy
         model.destroy!
       else
-        populate_model(model, message)
-        return if action == :update && !model.ps_subscriber_changed?(message)
+        populate_model(model)
+        return if action == :update && !model.ps_subscriber_changed?(payload.data)
 
         model.save!
       end
     end
 
-    def find_model(message)
+    def find_model
       model_class = klass.constantize
-      return model_class.ps_find_model(message) if model_class.respond_to?(:ps_find_model)
+      return model_class.ps_find_model(payload.data) if model_class.respond_to?(:ps_find_model)
 
-      model_class.where(model_identifiers(message)).first_or_initialize
+      model_class.where(model_identifiers).first_or_initialize
     end
 
-    def model_identifiers(message)
+    def model_identifiers
       identifiers = Array(settings[:id])
-      identifiers.map { |key| [key, message[key.to_sym]] }.to_h
+      identifiers.map { |key| [key, payload.data[key.to_sym]] }.to_h
     end
 
-    def populate_model(model, message)
-      values = message.slice(*attrs)
+    def populate_model(model)
+      values = payload.data.slice(*attrs)
       values.each do |attr, value|
         model.send("#{attr}=", value)
       end

data/lib/pub_sub_model_sync/subscriber_concern.rb

@@ -12,6 +12,10 @@ module PubSubModelSync
       changed?
     end
 
+    # permit to apply custom actions before applying sync
+    # @return (nil|:cancel): nil to continue sync OR :cancel to skip sync
+    def ps_before_save_sync(_payload); end
+
     module ClassMethods
       def ps_subscribe(attrs, actions: nil, from_klass: name, id: :id)
         settings = { id: id, from_klass: from_klass }

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.0'
+  VERSION = '0.5.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.3
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-12-22 00:00:00.000000000 Z
+date: 2020-12-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails