pub_sub_model_sync 1.1.1 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 84f478587ccd28d92b653744e72d338cf53818e5286544ff2c6e1e982a5a12a7
-  data.tar.gz: 32266d8a3ba69cad2e716c4861914b62e87ee96cf226ecf8f67dab3f909d9856
+  metadata.gz: 2df220f5f010c22b60791382718383661ae2d4200bb63d4cf3e03a2b096cc740
+  data.tar.gz: 84244d1c98800f15d54e3fbfdd3897fa05aea6f7285a8da5f1fcfb67d61513be
 SHA512:
-  metadata.gz: 02a727027a13d0b55f330395c95f9077568efb74fae3927c941211a4d849910fd6c071dae5fdd9386b3bb34c5257389cd74e4fcf5979e29f665a9b8e9e93aa95
-  data.tar.gz: 896f7430fbc83640ef3141a9e24413de69034ebc2d9e6a66987d734e25e9c41e52f03aa23faa494d9f05d0e4628acd058082e729a1de00c513fb0fb1c2cca70e
+  metadata.gz: eb2df15c0f909204e165b6d4e69d97434fcd50bc3ce3169086840753879b5ca0bf4de3942da0354e2528a203563084258ab3c6b003969336ae12311105e3a316
+  data.tar.gz: d3f4660cc54518d11bb66880a9c99100eb9e757cc9c1087b8a4a59cff976178d3f9d5fe97beddac55395b392083a381df9828a8dace92087f129a7130265759f

data/.github/workflows/ruby.yml

@@ -12,11 +12,15 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.4, 2.5, 2.6]
+        ruby: [2.5, 2.6]
         rails: [4, 5, 6]
         include:
           - ruby: 2.7
             rails: 6
+          - ruby: '3.0'
+            rails: 6
+          - ruby: '3.0'
+            rails: 7
         exclude: # rails 6 requires ruby >= 2.5
           - ruby: 2.4
             rails: 6

data/CHANGELOG.md

@@ -1,7 +1,13 @@
 # Change Log
 
+# 1.2.1 (October 28, 2021)
+chore: improve logs
+
+# 1.2.0 (October 28, 2021)
+- feat: rename Payload `:key` into `:internal_key` to avoid confusions while debugging
+
 # 1.1.1 (October 25, 2021)
-- feat: include `ordering_key topic_name` when delivering a notification for debugging purposes
+- feat: do not exclude `ordering_key topic_name` when delivering a notification (required when debugging)
 - doc: improve docs
 
 # 1.1.0 (October 25, 2021)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.1.1)
+    pub_sub_model_sync (1.3.1)
       rails
 
 GEM

data/README.md

@@ -311,14 +311,14 @@ Any notification before delivering is transformed as a Payload for a better port
     - `klass`: (String) Notification class name
     - `mode`: (Symbol: `:model`|`:class`) Kind of notification
   * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered. 
-    - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.         
-    - `key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).    
-      Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
+    - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.    
+      Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`         
+    - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).    
     - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
     - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
     - `app_key`: (Auto calculated): Name of the application who delivered the notification.   
     - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).   
-  Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info). 
+Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info). 
   
 - Actions
   ```ruby
@@ -343,21 +343,21 @@ Any notification before delivering is transformed as a Payload for a better port
       ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id user_id title]) }
     end
     ```
-    - When created (all notifications use the same ordering key to be processed in the same order)
+    - When created (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user = User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
         # notification #1 => <Payload data: {id: 1, name: 'sample'}, info: { klass: 'User', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When updated (all notifications use the same ordering key to be processed in the same order)
+    - When updated (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user.update!(name: 'changed', posts_attributes: [{ id: 1, title: 'Post 1C' }, { id: 2, title: 'Post 2C' }])
         # notification #1 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When destroyed (all notifications use the same ordering key to be processed in the same order)   
+    - When destroyed (all notifications use the same ordering_key to be processed in the same order)   
       **Note**: The notifications order were reordered in order to avoid inconsistency in other apps 
       ```ruby
       user.destroy!
@@ -371,7 +371,7 @@ Any notification before delivering is transformed as a Payload for a better port
   
   - Manual transactions   
     `PubSubModelSync::MessagePublisher::transaction(key, max_buffer: , &block)`
-    - `key` (String|nil) Key used as the ordering key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
+    - `key` (String|nil) Key used as the ordering_key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
     - `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (more details in #transactions_max_buffer).     
     Sample:
     ```ruby
@@ -386,37 +386,15 @@ Any notification before delivering is transformed as a Payload for a better port
 ## **Testing with RSpec**
 - Config: (spec/rails_helper.rb)
   ```ruby
-
-      # when using google service
-      require 'pub_sub_model_sync/mock_google_service'
-      config.before(:each) do
-        google_mock = PubSubModelSync::MockGoogleService.new
-        allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
-      end
-
-      # when using rabbitmq service
-      require 'pub_sub_model_sync/mock_rabbit_service'
-      config.before(:each) do
-        rabbit_mock = PubSubModelSync::MockRabbitService.new
-        allow(Bunny).to receive(:new).and_return(rabbit_mock)
-      end
-
-      # when using apache kafka service
-      require 'pub_sub_model_sync/mock_kafka_service'
-      config.before(:each) do
-        kafka_mock = PubSubModelSync::MockKafkaService.new
-        allow(Kafka).to receive(:new).and_return(kafka_mock)
-      end
-  
-      # disable all models sync by default (reduces testing time) 
       config.before(:each) do
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_data) # disable class level notif
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_model) # disable instance level notif
+        # disable delivering notifications to pubsub
+        allow(PubSubModelSync::MessagePublisher).to receive(:connector_publish)
+        # disable all models sync by default (reduces testing time by avoiding to build payload data)
+        allow(PubSubModelSync::MessagePublisher).to receive(:publish_model)
       end
     
       # enable all models sync only for tests that includes 'sync: true'
       config.before(:each, sync: true) do
-        allow(PubSubModelSync::MessagePublisher).to receive(:publish_data).and_call_original
         allow(PubSubModelSync::MessagePublisher).to receive(:publish_model).and_call_original
       end
       
@@ -426,9 +404,9 @@ Any notification before delivering is transformed as a Payload for a better port
       # end
   ```
 - Examples:
-  - **Publisher**
+  - **Publisher**    
+    Note: **Do not forget to include 'sync: true'** to enable publishing pubsub notifications
     ```ruby
-      # Do not forget to include 'sync: true' to enable publishing pubsub notifications
       describe 'When publishing sync', truncate: true, sync: true do
         it 'publishes user notification when created' do
           expect_publish_notification(:create, klass: 'User')
@@ -557,6 +535,7 @@ config.debug = true
     Note2: Only notifications from the buffer can be rollbacked if the current transaction has failed.     
 
 ## **TODO**
+- add the ability to raise SKIP_ACKNOWLEDGE to auto retry by PubSub
 - Auto publish update only if payload has changed (see ways to compare previous payload vs new payload)
 - Improve transactions to exclude similar notifications by klass and action. Sample:
     ```PubSubModelSync::MessagePublisher.transaction(key, { same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })```
@@ -568,6 +547,7 @@ config.debug = true
 - Add if/unless to ps_after_action
 - Add subscription liveness checker using thread without db connection to check periodically pending notifications from google pubsub
 - Unify .stop() and 'Listener stopped' 
+- TODO: Publish new version 1.2.1 (improve logs)
 
 ## **Q&A**
 - I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?

data/Rakefile

@@ -4,3 +4,4 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+# by default skip ACK when failed in three of them

data/gemfiles/Gemfile_4

@@ -5,7 +5,6 @@ gem 'bunny' # rabbit-mq
 gem 'google-cloud-pubsub' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 gem 'rails', '~> 4'
-gem 'bundler'
 gem 'sqlite3', '1.3.13'
 
 group :test do

data/gemfiles/Gemfile_7

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem 'rubocop'
+gem 'bunny' # rabbit-mq
+gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
+gem 'rails', '~> 7'
+
+group :test do
+  gem 'database_cleaner-active_record'
+end
+
+# Specify your gem's dependencies in pub_sub_model_sync.gemspec
+gemspec

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -34,7 +34,7 @@ module PubSubModelSync
       # @param key (String, Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
-        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        new_transaction = PubSubModelSync::Transaction.new(key, **settings)
         if current_transaction
           current_transaction.add_transaction(new_transaction)
         else

data/lib/pub_sub_model_sync/payload.rb

@@ -86,7 +86,7 @@ module PubSubModelSync
 
     def build_headers
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
-      headers[:key] ||= [klass, action].join('/')
+      headers[:internal_key] ||= [klass, action].join('/')
       headers[:ordering_key] ||= klass
       headers[:uuid] ||= SecureRandom.uuid
     end

data/lib/pub_sub_model_sync/payload_builder.rb

@@ -43,8 +43,8 @@ module PubSubModelSync
 
     def headers_data
       klass_name = model.class.name
-      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
-      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      internal_key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), internal_key: internal_key }
       def_data.merge(compute_value(headers))
     end
 

data/lib/pub_sub_model_sync/service_google.rb

@@ -77,12 +77,12 @@ module PubSubModelSync
       }.merge(PUBLISH_SETTINGS)
     end
 
-    # @return [Subscriber]
+    # @return [Array<Subscriber>]
     def subscribe_to_topics
       topics.map do |key, topic|
         subs_name = "#{config.subscription_key}_#{key}"
-        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, SUBSCRIPTION_SETTINGS)
-        subscriber = subscription.listen(LISTEN_SETTINGS, &method(:process_message))
+        subscription = topic.subscription(subs_name) || topic.subscribe(subs_name, **SUBSCRIPTION_SETTINGS)
+        subscriber = subscription.listen(**LISTEN_SETTINGS, &method(:process_message))
         subscriber.start
         log("Subscribed to topic: #{topic.name} as: #{subs_name}")
         subscriber

data/lib/pub_sub_model_sync/transaction.rb

@@ -19,10 +19,11 @@ module PubSubModelSync
     # @param payload (Payload)
     def add_payload(payload)
       payloads << payload
-      log("Payload added to current transaction: #{payload.inspect}") if config.debug
+      print_log = config.debug && max_buffer > 1
+      log("Payload added to current transaction: #{payload.inspect}") if print_log
       return unless payloads.count >= max_buffer
 
-      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
+      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}") if print_log
       deliver_payloads
     end
 
@@ -60,14 +61,14 @@ module PubSubModelSync
     private
 
     def deliver_payloads
-      payloads.each do |payload|
-        begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
-          PUBLISHER_KLASS.connector_publish(payload)
-        rescue => e
-          PUBLISHER_KLASS.send(:notify_error, e, payload)
-        end
-      end
+      payloads.each(&method(:deliver_payload))
       self.payloads = []
     end
+
+    def deliver_payload(payload)
+      PUBLISHER_KLASS.connector_publish(payload)
+    rescue => e
+      PUBLISHER_KLASS.send(:notify_error, e, payload)
+    end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.1.1'
+  VERSION = '1.3.1'
 end

data/samples/README.md

@@ -2,13 +2,27 @@
 This is a sample to sync information between rails applications using RabbitMQ
 
 ## Installation
-* Create manually the required network to share rabbitMQ (just if not exist):   
+1. Create manually the required network to share rabbitMQ accross Rails applications (just if not exist):   
   ```docker network create shared_app_services```
   
-* Start RabbitMQ server   
+2. Start RabbitMQ server   
   ```cd samples/app1 && docker-compose up pubsub```
 
-* In another tab access to App1 to publish notifications (Wait for step 2)  
+3. In another tab access to App2 to listen notifications (Wait for step 2)      
+  - Access to the folder
+    `cd samples/app2`
+  
+  - Build docker and start listener (Received notifications will be printed here)       
+    ```docker-compose run listener```
+  
+  - Optional: Open another tab to access application to ensure synced data
+    ```docker-compose run listener bash -c "rails c```
+    ```ruby
+      user = User.last.inspect
+      user.posts.inspect
+    ```
+
+4. In another tab access to App1 to publish notifications (Wait for step 2)  
   - Access to the application    
     `cd samples/app1`    
   
@@ -33,18 +47,4 @@ This is a sample to sync information between rails applications using RabbitMQ
     ```ruby
       user.destroy!
     ```    
-
-* In another tab access to App2 to listen notifications (Wait for step 2)      
-  - Access to the folder
-    `cd samples/app2`
-  
-  - Build docker and start listener (Received notifications will be printed here)       
-    ```docker-compose run listener```
   
-  - Optional: Open another tab to access application to ensure synced data
-    ```docker-compose run listener bash -c "rails c```
-    ```ruby
-      user = User.last.inspect
-      user.posts.inspect
-    ```
-  

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.3.1
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-28 00:00:00.000000000 Z
+date: 2022-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -107,6 +107,7 @@ files:
 - gemfiles/Gemfile_4
 - gemfiles/Gemfile_5
 - gemfiles/Gemfile_6
+- gemfiles/Gemfile_7
 - lib/pub_sub_model_sync.rb
 - lib/pub_sub_model_sync/base.rb
 - lib/pub_sub_model_sync/config.rb