pub_sub_model_sync 1.1.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 815981bccbd5dd0a03a603a2883b18c94bc321b37ed44623df65635ba645ea6f
-  data.tar.gz: b6e1bc85b69ce30df24f27726bb6e9950699799ddb67b313d8d9434284783f28
+  metadata.gz: e02077ac63ab98c674a0071b440c77da2e269109d8768c738b82362379e2c921
+  data.tar.gz: 0d29b69fb65290be2a7a437b87b04d53e4c8a394c98775eb1c98e20d96936866
 SHA512:
-  metadata.gz: e7f94f4d387baa8143f2fd2686a739bebefe12c247c48e6e85a3c7880efecaf02d5f01eb1770627cb06dbaeb9251f39a0052d715278c57e2c2771ca5fa342f61
-  data.tar.gz: 54a48c91cfc4cea1dd1e40583f9ac737dc324c0007e3bd8acb88599bb0322d1e3535c8d2dfe24d9023ebd51bd4abe9e7687974ee7d17028161757d267ce538b3
+  metadata.gz: 7e8877f20626a3404f8bb13a897d9e4d5fc28761a18e1cd88253d6210c323d25ac3c793c742ededb2ca11c5500b6997d5cf2c176300b444528ee206f6fdf3f0b
+  data.tar.gz: ae8f2610ee4778746b11781cd2f3b1e2bc97352989cc300cbc95b5884568b73f8f9f431d0d2c36d64f720fc777c21897e3f9e579d38e1b622da860dff8a2d75c

data/.github/workflows/ruby.yml

@@ -12,14 +12,20 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        ruby: [2.4, 2.5, 2.6]
+        ruby: [2.5, 2.6]
         rails: [4, 5, 6]
         include:
           - ruby: 2.7
             rails: 6
+          - ruby: '3.0'
+            rails: 6
+          - ruby: '3.0'
+            rails: 7
         exclude: # rails 6 requires ruby >= 2.5
           - ruby: 2.4
             rails: 6
+          - ruby: '3.0'
+            rails: 7
 
     steps:
       - uses: actions/checkout@v2

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Change Log
 
+# 1.2.1 (October 28, 2021)
+chore: improve logs
+
+# 1.2.0 (October 28, 2021)
+- feat: rename Payload `:key` into `:internal_key` to avoid confusions while debugging
+
+# 1.1.1 (October 25, 2021)
+- feat: do not exclude `ordering_key topic_name` when delivering a notification (required when debugging)
+- doc: improve docs
+
 # 1.1.0 (October 25, 2021)
 - feat: change `transactions_max_buffer` default value to 1 to deliver notifications once they were called
 - feat: use `after_commit` instead of `before_commit` callback and remove the horrible AR patch for rails 4

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.1.0)
+    pub_sub_model_sync (1.2.1)
       rails
 
 GEM
@@ -99,8 +99,8 @@ GEM
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
       googleauth (~> 0.15, >= 0.15.1)
       grpc (~> 1.36)
-    globalid (0.5.2)
-      activesupport (>= 5.0)
+    globalid (0.4.2)
+      activesupport (>= 4.2.0)
     google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
@@ -139,27 +139,27 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     jwt (2.2.3)
-    loofah (2.12.0)
+    loofah (2.9.1)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (1.0.2)
+    marcel (1.0.1)
     memoist (0.16.2)
     method_source (1.0.0)
     mini_mime (1.0.3)
     minitest (5.14.4)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    nio4r (2.5.8)
-    nokogiri (1.12.5-x86_64-linux)
+    nio4r (2.5.7)
+    nokogiri (1.11.3-x86_64-darwin)
       racc (~> 1.4)
     os (1.1.1)
     parallel (1.20.1)
     parser (3.0.1.1)
       ast (~> 2.4.1)
     public_suffix (4.0.6)
-    racc (1.6.0)
+    racc (1.5.2)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -181,7 +181,7 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.4.2)
+    rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
     railties (6.1.3.2)
       actionpack (= 6.1.3.2)
@@ -238,7 +238,7 @@ GEM
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
-    websocket-driver (0.7.5)
+    websocket-driver (0.7.3)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)

data/README.md

@@ -311,11 +311,14 @@ Any notification before delivering is transformed as a Payload for a better port
     - `klass`: (String) Notification class name
     - `mode`: (Symbol: `:model`|`:class`) Kind of notification
   * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered. 
-    - `key`: (String, optional) identifier of the payload, default: `<klass_name>/<action>` when class message, `<model.class.name>/<action>/<model.id>` when model message (Useful for caching techniques).
-    - `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message.     
-      Note: Final `ordering_key` is calculated by this way: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
-    - `topic_name`: (String|Array<String>, optional): Specific topic name (can be seen as a channel) to be used when delivering the message (default first topic from config).
-    - `forced_ordering_key`: (String, optional): Will force to use this value as the `ordering_key`, even withing transactions. Default `nil`.
+    - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.    
+      Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`         
+    - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).    
+    - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
+    - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
+    - `app_key`: (Auto calculated): Name of the application who delivered the notification.   
+    - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).   
+Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info). 
   
 - Actions
   ```ruby
@@ -340,21 +343,21 @@ Any notification before delivering is transformed as a Payload for a better port
       ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id user_id title]) }
     end
     ```
-    - When created (all notifications use the same ordering key to be processed in the same order)
+    - When created (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user = User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
         # notification #1 => <Payload data: {id: 1, name: 'sample'}, info: { klass: 'User', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When updated (all notifications use the same ordering key to be processed in the same order)
+    - When updated (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user.update!(name: 'changed', posts_attributes: [{ id: 1, title: 'Post 1C' }, { id: 2, title: 'Post 2C' }])
         # notification #1 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When destroyed (all notifications use the same ordering key to be processed in the same order)   
+    - When destroyed (all notifications use the same ordering_key to be processed in the same order)   
       **Note**: The notifications order were reordered in order to avoid inconsistency in other apps 
       ```ruby
       user.destroy!
@@ -368,7 +371,7 @@ Any notification before delivering is transformed as a Payload for a better port
   
   - Manual transactions   
     `PubSubModelSync::MessagePublisher::transaction(key, max_buffer: , &block)`
-    - `key` (String|nil) Key used as the ordering key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
+    - `key` (String|nil) Key used as the ordering_key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
     - `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (more details in #transactions_max_buffer).     
     Sample:
     ```ruby
@@ -528,7 +531,7 @@ config.debug = true
     Topic name(s) to be used to listen all notifications from when listening. Additionally first topic name is used as the default topic name when publishing a notification. 
 - `.subscription_name = "my-app-1"`:  (String, default Rails.application.name)     
     Subscriber's identifier which helps to: 
-    * skip self messages
+    * skip self notifications
     * continue the sync from the last synced notification when service was restarted.
 - `.default_topic_name = "my_topic"`: (String|Array<String>, optional(default first topic from `topic_name`))     
     Topic name used as the default topic if not defined in the payload when publishing a notification
@@ -537,25 +540,26 @@ config.debug = true
 - ```.logger = Rails.logger```
     (Logger) => define custom logger
 - ```.on_before_processing = ->(payload, {subscriber:}) { puts payload }```
-    (Proc) => called before processing received message (:cancel can be returned to skip processing)
+    (Proc) => called before processing a received notification (:cancel can be returned to skip processing)
 - ```.on_success_processing = ->(payload, {subscriber:}) { puts payload }```
-    (Proc) => called when a message was successfully processed
+    (Proc) => called when a notification was successfully processed
 - ```.on_error_processing = ->(exception, {payload:, subscriber:}) { payload.delay(...).process! }```
-    (Proc) => called when a message failed when processing (delayed_job or similar can be used for retrying)
+    (Proc) => called when a notification has failed when processing (delayed_job or similar can be used for retrying)
 - ```.on_before_publish = ->(payload) { puts payload }```
-    (Proc) => called before publishing a message (:cancel can be returned to skip publishing)
+    (Proc) => called before publishing a notification (:cancel can be returned to skip publishing)
 - ```.on_after_publish = ->(payload) { puts payload }```
-    (Proc) => called after publishing a message
+    (Proc) => called after publishing a notification
 - ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```
-    (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
+    (Proc) => called when failed publishing a notification (delayed_job or similar can be used for retrying)
 - ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.        
     Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).    
     Note: There is no way to rollback delivered notifications if current transaction fails later.      
     Note2: Only notifications from the buffer can be rollbacked if the current transaction has failed.     
 
 ## **TODO**
+- add the ability to raise SKIP_ACKNOWLEDGE to auto retry by PubSub
 - Auto publish update only if payload has changed (see ways to compare previous payload vs new payload)
-- Improve transactions to exclude similar messages by klass and action. Sample:
+- Improve transactions to exclude similar notifications by klass and action. Sample:
     ```PubSubModelSync::MessagePublisher.transaction(key, { same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })```
 - Add DB table to use as a shield to prevent publishing similar notifications and publish partial notifications (similar idea when processing notif)
 - Last notification is not being delivered immediately in google pubsub (maybe force with timeout 10secs and service.deliver_messages)
@@ -563,8 +567,9 @@ config.debug = true
 - Services support to deliver multiple payloads from transactions
 - Fix deprecation warnings: pub_sub_model_sync/service_google.rb:39: warning: Splitting the last argument into positional and keyword parameters is deprecated
 - Add if/unless to ps_after_action
-- Add subscription liveness checker using thread without db connection to check periodically pending messages from google pubsub
+- Add subscription liveness checker using thread without db connection to check periodically pending notifications from google pubsub
 - Unify .stop() and 'Listener stopped' 
+- TODO: Publish new version 1.2.1 (improve logs)
 
 ## **Q&A**
 - I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?

data/gemfiles/Gemfile_4

@@ -5,7 +5,6 @@ gem 'bunny' # rabbit-mq
 gem 'google-cloud-pubsub' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 gem 'rails', '~> 4'
-gem 'bundler'
 gem 'sqlite3', '1.3.13'
 
 group :test do

data/gemfiles/Gemfile_7

@@ -0,0 +1,14 @@
+source "https://rubygems.org"
+
+gem 'rubocop'
+gem 'bunny' # rabbit-mq
+gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
+gem 'rails', '~> 7'
+
+group :test do
+  gem 'database_cleaner-active_record'
+end
+
+# Specify your gem's dependencies in pub_sub_model_sync.gemspec
+gemspec

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -31,10 +31,10 @@ module PubSubModelSync
       end
 
       # Starts a new transaction
-      # @param key (String|Nil)
+      # @param key (String, Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
-        new_transaction = PubSubModelSync::Transaction.new(key, settings)
+        new_transaction = PubSubModelSync::Transaction.new(key, **settings)
         if current_transaction
           current_transaction.add_transaction(new_transaction)
         else

data/lib/pub_sub_model_sync/payload.rb

@@ -86,7 +86,7 @@ module PubSubModelSync
 
     def build_headers
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
-      headers[:key] ||= [klass, action].join('/')
+      headers[:internal_key] ||= [klass, action].join('/')
       headers[:ordering_key] ||= klass
       headers[:uuid] ||= SecureRandom.uuid
     end

data/lib/pub_sub_model_sync/payload_builder.rb

@@ -43,8 +43,8 @@ module PubSubModelSync
 
     def headers_data
       klass_name = model.class.name
-      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
-      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      internal_key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), internal_key: internal_key }
       def_data.merge(compute_value(headers))
     end
 

data/lib/pub_sub_model_sync/service_base.rb

@@ -24,7 +24,7 @@ module PubSubModelSync
     # @return (String): Json Format
     def encode_payload(payload)
       data = payload.to_h
-      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      not_important_keys = %i[forced_ordering_key]
       reduce_payload_size = !config.debug
       data[:headers].except!(*not_important_keys) if reduce_payload_size
       data.to_json

data/lib/pub_sub_model_sync/service_google.rb

@@ -77,7 +77,7 @@ module PubSubModelSync
       }.merge(PUBLISH_SETTINGS)
     end
 
-    # @return [Subscriber]
+    # @return [Array<Subscriber>]
     def subscribe_to_topics
       topics.map do |key, topic|
         subs_name = "#{config.subscription_key}_#{key}"

data/lib/pub_sub_model_sync/transaction.rb

@@ -5,7 +5,7 @@ module PubSubModelSync
     PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
     attr_accessor :key, :payloads, :max_buffer, :root, :children, :finished
 
-    # @param key (String,Null) Transaction key, if empty will use the ordering_key from first payload
+    # @param key (String,Nil) Transaction key, if empty will use the ordering_key from first payload
     # @param max_buffer (Integer) Once this quantity of notifications is reached, then all notifications
     #   will immediately be delivered.
     #   Note: There is no way to rollback delivered notifications if current transaction fails
@@ -19,10 +19,11 @@ module PubSubModelSync
     # @param payload (Payload)
     def add_payload(payload)
       payloads << payload
-      log("Payload added to current transaction: #{payload.inspect}") if config.debug
+      print_log = config.debug && max_buffer > 1
+      log("Payload added to current transaction: #{payload.inspect}") if print_log
       return unless payloads.count >= max_buffer
 
-      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
+      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}") if print_log
       deliver_payloads
     end
 
@@ -60,14 +61,14 @@ module PubSubModelSync
     private
 
     def deliver_payloads
-      payloads.each do |payload|
-        begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
-          PUBLISHER_KLASS.connector_publish(payload)
-        rescue => e
-          PUBLISHER_KLASS.send(:notify_error, e, payload)
-        end
-      end
+      payloads.each(&method(:deliver_payload))
       self.payloads = []
     end
+
+    def deliver_payload(payload)
+      PUBLISHER_KLASS.connector_publish(payload)
+    rescue => e
+      PUBLISHER_KLASS.send(:notify_error, e, payload)
+    end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.1.0'
+  VERSION = '1.3.0'
 end

data/samples/README.md

@@ -2,13 +2,27 @@
 This is a sample to sync information between rails applications using RabbitMQ
 
 ## Installation
-* Create manually the required network to share rabbitMQ (just if not exist):   
+1. Create manually the required network to share rabbitMQ accross Rails applications (just if not exist):   
   ```docker network create shared_app_services```
   
-* Start RabbitMQ server   
+2. Start RabbitMQ server   
   ```cd samples/app1 && docker-compose up pubsub```
 
-* In another tab access to App1 to publish notifications (Wait for step 2)  
+3. In another tab access to App2 to listen notifications (Wait for step 2)      
+  - Access to the folder
+    `cd samples/app2`
+  
+  - Build docker and start listener (Received notifications will be printed here)       
+    ```docker-compose run listener```
+  
+  - Optional: Open another tab to access application to ensure synced data
+    ```docker-compose run listener bash -c "rails c```
+    ```ruby
+      user = User.last.inspect
+      user.posts.inspect
+    ```
+
+4. In another tab access to App1 to publish notifications (Wait for step 2)  
   - Access to the application    
     `cd samples/app1`    
   
@@ -33,18 +47,4 @@ This is a sample to sync information between rails applications using RabbitMQ
     ```ruby
       user.destroy!
     ```    
-
-* In another tab access to App2 to listen notifications (Wait for step 2)      
-  - Access to the folder
-    `cd samples/app2`
-  
-  - Build docker and start listener (Received notifications will be printed here)       
-    ```docker-compose run listener```
   
-  - Optional: Open another tab to access application to ensure synced data
-    ```docker-compose run listener bash -c "rails c```
-    ```ruby
-      user = User.last.inspect
-      user.posts.inspect
-    ```
-  

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-10-25 00:00:00.000000000 Z
+date: 2022-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -107,6 +107,7 @@ files:
 - gemfiles/Gemfile_4
 - gemfiles/Gemfile_5
 - gemfiles/Gemfile_6
+- gemfiles/Gemfile_7
 - lib/pub_sub_model_sync.rb
 - lib/pub_sub_model_sync/base.rb
 - lib/pub_sub_model_sync/config.rb