pub_sub_model_sync 1.2.1 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86dd60cd72d095630d44b3d1d6bb0004f0d7ce35326f5613127815047376530a
-  data.tar.gz: ef7c8edd08a315adcbad4a4314a467e1564a8dcc2233764ef53d09985a41eaf3
+  metadata.gz: 1740843df35081aaeb35e2be5c1c27cf1ff4d219af0bcf9ae4394495ab8bba5e
+  data.tar.gz: ed506905c6b29b554f4a1cd100ddf4c8df2bf0a99590eb26901fae8feef862ce
 SHA512:
-  metadata.gz: 690ca42fe8d463cabdaa4e0e557ee1fa19ffa0e1c9a4ad19ee1a1195a7a78faab458ebcd0b0581c39b917ac73c9dc01f408bf1c9ae7f25445b7d20afba2592fe
-  data.tar.gz: 72d3e0a10a8879f670e65a6f50dbc71e0351893ac1b57dc6fe1796373f7a38d55ecc7a6508c86eba7256dab03c903a42b59f9a0b7a881f6c4c28086b59584049
+  metadata.gz: bff308ce421ae321d2a9e7b53df194d3678e5f6103eb4f2048d4ed064302c042fb7768f96a3c639190b8d452a0636fded9e72394a5ffe4731b4e24ba0b2f5b53
+  data.tar.gz: e60c4c78c5fbde7c40d68adb1ed029d74bb10d1759a27682850aae6015c84f938cbccccaadc976878afa26d97621d360937cdeb5cc028b2393facb12fa0d4831

data/.github/workflows/ruby.yml

@@ -12,11 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.4, 2.5, 2.6]
+        ruby: [2.5, 2.6]
         rails: [4, 5, 6]
         include:
           - ruby: 2.7
             rails: 6
+          - ruby: '3.0'
+            rails: 6
+          - ruby: '3.0'
+            rails: 7
         exclude: # rails 6 requires ruby >= 2.5
           - ruby: 2.4
             rails: 6

data/CHANGELOG.md

@@ -1,5 +1,7 @@
 # Change Log
 
+For recent releases list go to: https://github.com/owen2345/pub_sub_model_sync/releases
+
 # 1.2.1 (October 28, 2021)
 chore: improve logs
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.2.1)
+    pub_sub_model_sync (1.4.0)
       rails
 
 GEM

data/README.md

@@ -313,10 +313,15 @@ Any notification before delivering is transformed as a Payload for a better port
   * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered. 
     - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.    
       Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`         
-    - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).    
     - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
     - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
-    - `app_key`: (Auto calculated): Name of the application who delivered the notification.   
+    - `cache` (Boolean | Hash, Default false) Cache settings   
+        - `true`: Skip publishing similar payloads
+        - `Hash<required: Array<Symbol>>`: Same as `true` and enables payload optimization to exclude unchanged non important attributes. Sample: `{ required: %i[id email] }`
+    
+    ** Read ONLY **    
+    - `internal_key`: Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).
+    - `app_key`: (Auto calculated): Name of the application who delivered the notification.
     - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).   
 Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info). 
   
@@ -386,37 +391,15 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
 ## **Testing with RSpec**
 - Config: (spec/rails_helper.rb)
   ```ruby
-
-      # when using google service
-      require 'pub_sub_model_sync/mock_google_service'
-      config.before(:each) do
-        google_mock = PubSubModelSync::MockGoogleService.new
-        allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
-      end
-
-      # when using rabbitmq service
-      require 'pub_sub_model_sync/mock_rabbit_service'
-      config.before(:each) do
-        rabbit_mock = PubSubModelSync::MockRabbitService.new
-        allow(Bunny).to receive(:new).and_return(rabbit_mock)
-      end
-
-      # when using apache kafka service
-      require 'pub_sub_model_sync/mock_kafka_service'
-      config.before(:each) do
-        kafka_mock = PubSubModelSync::MockKafkaService.new
-        allow(Kafka).to receive(:new).and_return(kafka_mock)
-      end
-  
-      # disable all models sync by default (reduces testing time) 
       config.before(:each) do
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
+        # disable delivering notifications to pubsub
+        allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
+        # disable all models sync by default (reduces testing time by avoiding to build payload data)
+        allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
       end
     
       # enable all models sync only for tests that includes 'sync: true'
       config.before(:each, sync: true) do
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
         allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
       end
       
@@ -426,9 +409,9 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
       # end
   ```
 - Examples:
-  - **Publisher**
+  - **Publisher**    
+    Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
     ```ruby
-      # Do not forget to include 'sync: true' to enable publishing pubsub notifications
       describe 'When publishing sync', truncate: true, sync: true do
         it 'publishes user notification when created' do
           expect_publish_notification(:create, klass: 'User')

data/Rakefile

@@ -4,3 +4,4 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+# by default skip ACK when failed in three of them

data/gemfiles/Gemfile_4

@@ -5,7 +5,6 @@ gem 'bunny' # rabbit-mq
 gem 'google-cloud-pubsub' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 gem 'rails', '~> 4'
-gem 'bundler'
 gem 'sqlite3', '1.3.13'
 
 group :test do

data/gemfiles/Gemfile_7

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem 'rubocop'
+gem 'bunny' # rabbit-mq
+gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
+gem 'rails', '~> 7'
+
+group :test do
+  gem 'database_cleaner-active_record'
+end
+
+# Specify your gem's dependencies in pub_sub_model_sync.gemspec
+gemspec

data/lib/pub_sub_model_sync/config.rb

@@ -9,6 +9,7 @@ module PubSubModelSync
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
     cattr_accessor(:transactions_max_buffer) { 1 }
+    cattr_accessor(:skip_cache) { false }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -34,7 +34,7 @@ module PubSubModelSync
       # @param key (String, Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
-        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        new_transaction = PubSubModelSync::Transaction.new(key, **settings)
         if current_transaction
           current_transaction.add_transaction(new_transaction)
         else
@@ -98,8 +98,11 @@ module PubSubModelSync
       private
 
       def ensure_publish(payload)
-        cancelled = config.on_before_publish.call(payload) == :cancel
-        log("Publish cancelled by config.on_before_publish: #{[payload]}") if config.debug && cancelled
+        cache_klass = PubSubModelSync::PayloadCacheOptimizer
+        cancelled = payload.cache_settings ? cache_klass.new(payload).call == :already_sent : false
+        cancelled ||= config.on_before_publish.call(payload) == :cancel
+        log_msg = "Publish cancelled by config.on_before_publish or cache checker: #{[payload]}"
+        log(log_msg) if config.debug && cancelled
         !cancelled
       end
 

data/lib/pub_sub_model_sync/payload.rb

@@ -19,9 +19,17 @@ module PubSubModelSync
     #        <klass>: when class message
     #        <klass/id>: when model message
     #   topic_name (String|Array<String>): Specific topic name to be used when delivering the
-    #     message (default first topic)
+    #     message (default Config.topic_name)
     #   forced_ordering_key (String, optional): Will force to use this value as the ordering_key,
     #     even withing transactions. Default nil.
+    #   cache (Boolean | Hash, Default false) Cache settings
+    #     true: Skip publishing similar payloads
+    #     Hash<required: Array<Symbol>>: Same as true and enables payload optimization to exclude
+    #       unchanged non important attributes. Sample: { required: %i[id email] }
+    #   --- READ ONLY ----
+    #   app_key: (string) Subscriber-Key of the application who delivered the notification
+    #   internal_key: (String) "<klass>/<action>"
+    #   uuid: Unique notification identifier
     def initialize(data, info, headers = {})
       @data = data.deep_symbolize_keys
       @info = info.deep_symbolize_keys
@@ -75,6 +83,16 @@ module PubSubModelSync
       klass.publish(self)
     end
 
+    # @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
+    def exclude_data_attrs(attr_keys)
+      @data = data.except(*attr_keys)
+    end
+
+    # Attributes to always be delivered after cache optimization
+    def cache_settings
+      headers[:cache]
+    end
+
     # convert payload data into Payload
     # @param data [Hash]: payload data (:data, :info, :headers)
     def self.from_payload_data(data)

data/lib/pub_sub_model_sync/payload_cache_optimizer.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class PayloadCacheOptimizer < PubSubModelSync::Base
+    # Optimizes payload data to deliver only the required ones and the changed ones and thus avoid
+    #   delivering unnecessary notifications.
+    #   Uses Rails.cache to retrieve previous delivered data.
+    attr_reader :payload, :required_attrs, :cache_key
+
+    # @param payload (Payload)
+    def initialize(payload)
+      @payload = payload
+      @cache_key = "pubsub/#{payload.headers[:internal_key]}/#{payload.headers[:topic_name]}"
+    end
+
+    # @return (:already_sent|Payload)
+    def call
+      backup_data = payload.data.clone
+      return payload if cache_disabled?
+      return :already_sent if previous_payload_data == payload.data
+
+      optimize_payload if optimization_enabled?
+      Rails.cache.write(cache_key, backup_data, expires_in: 1.week)
+      payload
+    end
+
+    private
+
+    def optimization_enabled?
+      previous_payload_data && payload.cache_settings.is_a?(Hash)
+    end
+
+    def cache_disabled?
+      res = config.skip_cache || Rails.cache.nil?
+      log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
+      res
+    end
+
+    def previous_payload_data
+      @previous_payload_data ||= Rails.cache.read(cache_key)
+    end
+
+    def optimize_payload # rubocop:disable Metrics/AbcSize
+      changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys
+      invalid_keys = payload.data.keys - (changed_keys + payload.cache_settings[:required])
+      log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
+      payload.exclude_data_attrs(invalid_keys)
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_base.rb

@@ -24,7 +24,7 @@ module PubSubModelSync
     # @return (String): Json Format
     def encode_payload(payload)
       data = payload.to_h
-      not_important_keys = %i[forced_ordering_key]
+      not_important_keys = %i[forced_ordering_key cache]
       reduce_payload_size = !config.debug
       data[:headers].except!(*not_important_keys) if reduce_payload_size
       data.to_json

data/lib/pub_sub_model_sync/service_google.rb

@@ -81,8 +81,8 @@ module PubSubModelSync
     def subscribe_to_topics
       topics.map do |key, topic|
         subs_name = "#{config.subscription_key}_#{key}"
-        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
-        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
         subscriber.start
         log("Subscribed to topic: #{topic.name} as: #{subs_name}")
         subscriber

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.2.1'
+  VERSION = '1.4.0'
 end

data/lib/pub_sub_model_sync.rb

@@ -16,6 +16,7 @@ require 'pub_sub_model_sync/message_processor'
 require 'pub_sub_model_sync/run_subscriber'
 
 require 'pub_sub_model_sync/payload_builder'
+require 'pub_sub_model_sync/payload_cache_optimizer'
 require 'pub_sub_model_sync/subscriber'
 
 require 'pub_sub_model_sync/service_base'

data/samples/README.md

@@ -2,13 +2,27 @@
 This is a sample to sync information between rails applications using RabbitMQ
 
 ## Installation
-* Create manually the required network to share rabbitMQ (just if not exist):   
+1. Create manually the required network to share rabbitMQ accross Rails applications (just if not exist):   
   ```docker network create shared_app_services```
   
-* Start RabbitMQ server   
+2. Start RabbitMQ server   
   ```cd samples/app1 && docker-compose up pubsub```
 
-* In another tab access to App1 to publish notifications (Wait for step 2)  
+3. In another tab access to App2 to listen notifications (Wait for step 2)      
+  - Access to the folder
+    `cd samples/app2`
+  
+  - Build docker and start listener (Received notifications will be printed here)       
+    ```docker-compose run listener```
+  
+  - Optional: Open another tab to access application to ensure synced data
+    ```docker-compose run listener bash -c "rails c```
+    ```ruby
+      user = User.last.inspect
+      user.posts.inspect
+    ```
+
+4. In another tab access to App1 to publish notifications (Wait for step 2)  
   - Access to the application    
     `cd samples/app1`    
   
@@ -33,18 +47,4 @@ This is a sample to sync information between rails applications using RabbitMQ
     ```ruby
       user.destroy!
     ```    
-
-* In another tab access to App2 to listen notifications (Wait for step 2)      
-  - Access to the folder
-    `cd samples/app2`
-  
-  - Build docker and start listener (Received notifications will be printed here)       
-    ```docker-compose run listener```
   
-  - Optional: Open another tab to access application to ensure synced data
-    ```docker-compose run listener bash -c "rails c```
-    ```ruby
-      user = User.last.inspect
-      user.posts.inspect
-    ```
-  

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.4.0
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2022-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -107,6 +107,7 @@ files:
 - gemfiles/Gemfile_4
 - gemfiles/Gemfile_5
 - gemfiles/Gemfile_6
+- gemfiles/Gemfile_7
 - lib/pub_sub_model_sync.rb
 - lib/pub_sub_model_sync/base.rb
 - lib/pub_sub_model_sync/config.rb
@@ -118,6 +119,7 @@ files:
 - lib/pub_sub_model_sync/mock_rabbit_service.rb
 - lib/pub_sub_model_sync/payload.rb
 - lib/pub_sub_model_sync/payload_builder.rb
+- lib/pub_sub_model_sync/payload_cache_optimizer.rb
 - lib/pub_sub_model_sync/publisher_concern.rb
 - lib/pub_sub_model_sync/railtie.rb
 - lib/pub_sub_model_sync/run_subscriber.rb