pub_sub_model_sync 1.0.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6e73c2af5a62d4b0ae32ea40f3c3c369a049edfd47360a50386eeb5e4ff97cd1
-  data.tar.gz: 5d1892ee41ada21c9bf70bcc3601e435edfdf55d7b51b443369bf3307da60436
+  metadata.gz: 86dd60cd72d095630d44b3d1d6bb0004f0d7ce35326f5613127815047376530a
+  data.tar.gz: ef7c8edd08a315adcbad4a4314a467e1564a8dcc2233764ef53d09985a41eaf3
 SHA512:
-  metadata.gz: 962defddfdc780ce185d01e9069485207bdf66be9b3554a1003b5ec71211c467223c288fbb6c6a59ae819a9ba37ff118a470942bdef9e69c68e94f34ff899c9a
-  data.tar.gz: c6e771a40a96dcdf00161aa49fbe7c1b5f283709d5b42b87421c6655e4cb9972e9f7d3bb7d203d47c9757dc681f8c755c7026447f6582ffa1f43a553ba90197a
+  metadata.gz: 690ca42fe8d463cabdaa4e0e557ee1fa19ffa0e1c9a4ad19ee1a1195a7a78faab458ebcd0b0581c39b917ac73c9dc01f408bf1c9ae7f25445b7d20afba2592fe
+  data.tar.gz: 72d3e0a10a8879f670e65a6f50dbc71e0351893ac1b57dc6fe1796373f7a38d55ecc7a6508c86eba7256dab03c903a42b59f9a0b7a881f6c4c28086b59584049

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Change Log
 
+# 1.2.1 (October 28, 2021)
+chore: improve logs
+
+# 1.2.0 (October 28, 2021)
+- feat: rename Payload `:key` into `:internal_key` to avoid confusions while debugging
+
+# 1.1.1 (October 25, 2021)
+- feat: do not exclude `ordering_key topic_name` when delivering a notification (required when debugging)
+- doc: improve docs
+
+# 1.1.0 (October 25, 2021)
+- feat: change `transactions_max_buffer` default value to 1 to deliver notifications once they were called
+- feat: use `after_commit` instead of `before_commit` callback and remove the horrible AR patch for rails 4
+
 # 1.0.1 (August 20, 2021)
 - refactor: improve service exit when running in k8s
 

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.0.1)
+    pub_sub_model_sync (1.2.1)
       rails
 
 GEM
@@ -116,7 +116,6 @@ GEM
       google-cloud-errors (~> 1.0)
       grpc-google-iam-v1 (>= 0.6.10, < 2.0)
     google-protobuf (3.17.0)
-    google-protobuf (3.17.0-x86_64-linux)
     googleapis-common-protos (1.3.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
@@ -133,9 +132,6 @@ GEM
     grpc (1.37.1)
       google-protobuf (~> 3.15)
       googleapis-common-protos-types (~> 1.0)
-    grpc (1.37.1-x86_64-linux)
-      google-protobuf (~> 3.15)
-      googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.11, < 2.0)
@@ -264,4 +260,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   2.2.17
+   2.2.29

data/README.md

@@ -245,7 +245,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
   - `method_name` (Symbol, optional) method to be called to process action callback, sample: `def my_method(action) ... end`
   - `block` (Proc, optional) Block to be called to process action callback, sample: `{ |action| ... }`     
   
-  **Note1**: Due to rails callback ordering, this method uses `before_commit` callback when creating or updating models to ensure expected notifications order (More details [**here**](#transactions)).    
+  **Note1**: Due to rails callback ordering, this method uses `after_commit on: action {...}` callback when creating or updating models to ensure expected notifications order (More details [**here**](#transactions)).    
   **Note2**: Due to rails callback ordering, this method uses `after_destroy` callback when destroying models to ensure the expected notifications order.    
    
 - `ps_publish(action, data: {}, mapping: [], headers: {}, as_klass: nil)` Delivers an instance notification via pubsub
@@ -311,11 +311,14 @@ Any notification before delivering is transformed as a Payload for a better port
     - `klass`: (String) Notification class name
     - `mode`: (Symbol: `:model`|`:class`) Kind of notification
   * `headers`: (Hash) Notification settings that defines how the notification will be processed or delivered. 
-    - `key`: (String, optional) identifier of the payload, default: `<klass_name>/<action>` when class message, `<model.class.name>/<action>/<model.id>` when model message (Useful for caching techniques).
-    - `ordering_key`: (String, optional): messages with the same value are processed in the same order they were delivered, default: `klass_name` when class message, `<model.class.name>/<model.id>` when instance message.     
-      Note: Final `ordering_key` is calculated by this way: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`
-    - `topic_name`: (String|Array<String>, optional): Specific topic name (can be seen as a channel) to be used when delivering the message (default first topic from config).
-    - `forced_ordering_key`: (String, optional): Will force to use this value as the `ordering_key`, even withing transactions. Default `nil`.
+    - `ordering_key`: (String, optional): notifications with the same `ordering_key` are processed in the same order they were delivered, default: `<model.class.name>/<model.id>` when instance notification and `klass_name` when class notification.    
+      Note: Final `ordering_key` is calculated as: `payload.headers[:forced_ordering_key] || current_transaction&.key || payload.headers[:ordering_key]`         
+    - `internal_key`: (String, optional) Internal identifier of the payload, default: `<model.class.name>/<action>/<model.id>` when model notification and `<klass_name>/<action>` when class notification (Useful for caching techniques).    
+    - `topic_name`: (String|Array<String>, optional): Specific topic name where to deliver the notification (default `PubSubModelSync::Config.topic_name`).
+    - `forced_ordering_key`: (String, optional): Overrides `ordering_key` with the provided value even withing transactions. Default `nil`.
+    - `app_key`: (Auto calculated): Name of the application who delivered the notification.   
+    - `uuid`: (Auto calculated): Unique notification identifier (Very useful when debugging).   
+Note: To reduce Payload size, some header info are not delivered (Enable debug mode to deliver all payload info). 
   
 - Actions
   ```ruby
@@ -340,21 +343,21 @@ Any notification before delivering is transformed as a Payload for a better port
       ps_after_action([:create, :update, :destroy]) { |action| ps_publish(action, mapping: %i[id user_id title]) }
     end
     ```
-    - When created (all notifications use the same ordering key to be processed in the same order)
+    - When created (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user = User.create!(name: 'test', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
         # notification #1 => <Payload data: {id: 1, name: 'sample'}, info: { klass: 'User', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2', user_id: 1}, info: { klass: 'Post', action: :create, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When updated (all notifications use the same ordering key to be processed in the same order)
+    - When updated (all notifications use the same ordering_key to be processed in the same order)
       ```ruby
         user.update!(name: 'changed', posts_attributes: [{ id: 1, title: 'Post 1C' }, { id: 2, title: 'Post 2C' }])
         # notification #1 => <Payload data: {id: 1, name: 'changed'}, info: { klass: 'User', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #2 => <Payload data: {id: 1, title: 'Post 1C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
         # notification #3 => <Payload data: {id: 2, title: 'Post 2C', user_id: 1}, info: { klass: 'Post', action: :update, mode: :model }, headers: { ordering_key = `User/1` }>
       ```
-    - When destroyed (all notifications use the same ordering key to be processed in the same order)   
+    - When destroyed (all notifications use the same ordering_key to be processed in the same order)   
       **Note**: The notifications order were reordered in order to avoid inconsistency in other apps 
       ```ruby
       user.destroy!
@@ -368,10 +371,8 @@ Any notification before delivering is transformed as a Payload for a better port
   
   - Manual transactions   
     `PubSubModelSync::MessagePublisher::transaction(key, max_buffer: , &block)`
-    - `key` (String|nil) Key used as the ordering key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
-    - `max_buffer:` (Boolean, default: `PubSubModelSync::Config.transactions_max_buffer`)     
-        If true: will save all notifications and deliver all them when transaction has successfully finished. If transaction has failed, then all saved notifications will be discarded (not delivered).    
-        If false: will deliver all notifications immediately (no way to rollback notifications if transaction has failed)  
+    - `key` (String|nil) Key used as the ordering_key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
+    - `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (more details in #transactions_max_buffer).     
     Sample:
     ```ruby
       PubSubModelSync::MessagePublisher::transaction('my-custom-key') do
@@ -530,7 +531,7 @@ config.debug = true
     Topic name(s) to be used to listen all notifications from when listening. Additionally first topic name is used as the default topic name when publishing a notification. 
 - `.subscription_name = "my-app-1"`:  (String, default Rails.application.name)     
     Subscriber's identifier which helps to: 
-    * skip self messages
+    * skip self notifications
     * continue the sync from the last synced notification when service was restarted.
 - `.default_topic_name = "my_topic"`: (String|Array<String>, optional(default first topic from `topic_name`))     
     Topic name used as the default topic if not defined in the payload when publishing a notification
@@ -539,24 +540,26 @@ config.debug = true
 - ```.logger = Rails.logger```
     (Logger) => define custom logger
 - ```.on_before_processing = ->(payload, {subscriber:}) { puts payload }```
-    (Proc) => called before processing received message (:cancel can be returned to skip processing)
+    (Proc) => called before processing a received notification (:cancel can be returned to skip processing)
 - ```.on_success_processing = ->(payload, {subscriber:}) { puts payload }```
-    (Proc) => called when a message was successfully processed
+    (Proc) => called when a notification was successfully processed
 - ```.on_error_processing = ->(exception, {payload:, subscriber:}) { payload.delay(...).process! }```
-    (Proc) => called when a message failed when processing (delayed_job or similar can be used for retrying)
+    (Proc) => called when a notification has failed when processing (delayed_job or similar can be used for retrying)
 - ```.on_before_publish = ->(payload) { puts payload }```
-    (Proc) => called before publishing a message (:cancel can be returned to skip publishing)
+    (Proc) => called before publishing a notification (:cancel can be returned to skip publishing)
 - ```.on_after_publish = ->(payload) { puts payload }```
-    (Proc) => called after publishing a message
+    (Proc) => called after publishing a notification
 - ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).publish! }```
-    (Proc) => called when failed publishing a message (delayed_job or similar can be used for retrying)
-- ```.transactions_max_buffer = 100``` (Integer) Once this quantity of notifications is reached, then all notifications will immediately be delivered.    
-    Note: There is no way to rollback delivered notifications if current transaction fails
-- ```.enable_rails4_before_commit = true``` (true*|false) When false will disable rails 4 hack compatibility and then CRUD notifications will be prepared using `after_commit` callback instead of `before_commit` (used in `ps_after_action(...)`) which will not rollback sql transactions if failed when publishing pubsub notification.
+    (Proc) => called when failed publishing a notification (delayed_job or similar can be used for retrying)
+- ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.        
+    Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).    
+    Note: There is no way to rollback delivered notifications if current transaction fails later.      
+    Note2: Only notifications from the buffer can be rollbacked if the current transaction has failed.     
 
 ## **TODO**
+- add the ability to raise SKIP_ACKNOWLEDGE to auto retry by PubSub
 - Auto publish update only if payload has changed (see ways to compare previous payload vs new payload)
-- Improve transactions to exclude similar messages by klass and action. Sample:
+- Improve transactions to exclude similar notifications by klass and action. Sample:
     ```PubSubModelSync::MessagePublisher.transaction(key, { same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })```
 - Add DB table to use as a shield to prevent publishing similar notifications and publish partial notifications (similar idea when processing notif)
 - Last notification is not being delivered immediately in google pubsub (maybe force with timeout 10secs and service.deliver_messages)
@@ -564,6 +567,9 @@ config.debug = true
 - Services support to deliver multiple payloads from transactions
 - Fix deprecation warnings: pub_sub_model_sync/service_google.rb:39: warning: Splitting the last argument into positional and keyword parameters is deprecated
 - Add if/unless to ps_after_action
+- Add subscription liveness checker using thread without db connection to check periodically pending notifications from google pubsub
+- Unify .stop() and 'Listener stopped' 
+- TODO: Publish new version 1.2.1 (improve logs)
 
 ## **Q&A**
 - I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?

data/lib/pub_sub_model_sync/config.rb

@@ -8,8 +8,7 @@ module PubSubModelSync
     # customizable callbacks
     cattr_accessor(:debug) { false }
     cattr_accessor :logger # LoggerInst
-    cattr_accessor(:transactions_max_buffer) { 100 }
-    cattr_accessor(:enable_rails4_before_commit) { Rails::VERSION::MAJOR == 4 }
+    cattr_accessor(:transactions_max_buffer) { 1 }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -31,7 +31,7 @@ module PubSubModelSync
       end
 
       # Starts a new transaction
-      # @param key (String|Nil)
+      # @param key (String, Nil)
       # @return (Transaction)
       def init_transaction(key, settings = {})
         new_transaction = PubSubModelSync::Transaction.new(key, settings)

data/lib/pub_sub_model_sync/payload.rb

@@ -86,7 +86,7 @@ module PubSubModelSync
 
     def build_headers
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
-      headers[:key] ||= [klass, action].join('/')
+      headers[:internal_key] ||= [klass, action].join('/')
       headers[:ordering_key] ||= klass
       headers[:uuid] ||= SecureRandom.uuid
     end

data/lib/pub_sub_model_sync/payload_builder.rb

@@ -43,8 +43,8 @@ module PubSubModelSync
 
     def headers_data
       klass_name = model.class.name
-      key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
-      def_data = { ordering_key: self.class.ordering_key_for(model), key: key }
+      internal_key = [klass_name, action, model.id || SecureRandom.uuid].join('/')
+      def_data = { ordering_key: self.class.ordering_key_for(model), internal_key: internal_key }
       def_data.merge(compute_value(headers))
     end
 

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -67,7 +67,7 @@ module PubSubModelSync
           if action == :destroy
             after_destroy { instance_exec(action, &callback) }
           else
-            ps_define_commit_action(action, callback)
+            send(:after_commit, on: action) { instance_exec(action, &callback) }
           end
         end
       end
@@ -80,15 +80,6 @@ module PubSubModelSync
 
       private
 
-      def ps_define_commit_action(action, callback)
-        if PubSubModelSync::Config.enable_rails4_before_commit # rails 4 compatibility
-          define_method("ps_before_#{action}_commit") { instance_exec(action, &callback) }
-        else
-          commit_name = respond_to?(:before_commit) ? :before_commit : :after_commit
-          send(commit_name, on: action) { instance_exec(action, &callback) }
-        end
-      end
-
       # Initialize calls to start and end pub_sub transactions and deliver all them in the same order
       def ps_init_transaction_callbacks
         start_transaction = lambda do

data/lib/pub_sub_model_sync/railtie.rb

@@ -13,7 +13,6 @@ module PubSubModelSync
     end
 
     configure do
-      require 'pub_sub_model_sync/initializers/before_commit' if PubSubModelSync::Config.enable_rails4_before_commit
     end
   end
 end

data/lib/pub_sub_model_sync/service_base.rb

@@ -24,7 +24,7 @@ module PubSubModelSync
     # @return (String): Json Format
     def encode_payload(payload)
       data = payload.to_h
-      not_important_keys = %i[ordering_key topic_name forced_ordering_key]
+      not_important_keys = %i[forced_ordering_key]
       reduce_payload_size = !config.debug
       data[:headers].except!(*not_important_keys) if reduce_payload_size
       data.to_json

data/lib/pub_sub_model_sync/service_google.rb

@@ -77,7 +77,7 @@ module PubSubModelSync
       }.merge(PUBLISH_SETTINGS)
     end
 
-    # @return [Subscriber]
+    # @return [Array<Subscriber>]
     def subscribe_to_topics
       topics.map do |key, topic|
         subs_name = "#{config.subscription_key}_#{key}"

data/lib/pub_sub_model_sync/transaction.rb

@@ -5,7 +5,7 @@ module PubSubModelSync
     PUBLISHER_KLASS = PubSubModelSync::MessagePublisher
     attr_accessor :key, :payloads, :max_buffer, :root, :children, :finished
 
-    # @param key (String,Null) Transaction key, if empty will use the ordering_key from first payload
+    # @param key (String,Nil) Transaction key, if empty will use the ordering_key from first payload
     # @param max_buffer (Integer) Once this quantity of notifications is reached, then all notifications
     #   will immediately be delivered.
     #   Note: There is no way to rollback delivered notifications if current transaction fails
@@ -19,10 +19,11 @@ module PubSubModelSync
     # @param payload (Payload)
     def add_payload(payload)
       payloads << payload
-      log("Payload added to current transaction: #{payload.inspect}") if config.debug
+      print_log = config.debug && max_buffer > 1
+      log("Payload added to current transaction: #{payload.inspect}") if print_log
       return unless payloads.count >= max_buffer
 
-      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}")
+      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}") if print_log
       deliver_payloads
     end
 
@@ -60,14 +61,14 @@ module PubSubModelSync
     private
 
     def deliver_payloads
-      payloads.each do |payload|
-        begin # rubocop:disable Style/RedundantBegin (ruby 2.4 support)
-          PUBLISHER_KLASS.connector_publish(payload)
-        rescue => e
-          PUBLISHER_KLASS.send(:notify_error, e, payload)
-        end
-      end
+      payloads.each(&method(:deliver_payload))
       self.payloads = []
     end
+
+    def deliver_payload(payload)
+      PUBLISHER_KLASS.connector_publish(payload)
+    rescue => e
+      PUBLISHER_KLASS.send(:notify_error, e, payload)
+    end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.0.1'
+  VERSION = '1.2.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.2.1
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-08-20 00:00:00.000000000 Z
+date: 2021-10-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -111,7 +111,6 @@ files:
 - lib/pub_sub_model_sync/base.rb
 - lib/pub_sub_model_sync/config.rb
 - lib/pub_sub_model_sync/connector.rb
-- lib/pub_sub_model_sync/initializers/before_commit.rb
 - lib/pub_sub_model_sync/message_processor.rb
 - lib/pub_sub_model_sync/message_publisher.rb
 - lib/pub_sub_model_sync/mock_google_service.rb

data/lib/pub_sub_model_sync/initializers/before_commit.rb

@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-# Rails 4 backward compatibility (Add "simple" ps_before_*_commit callbacks)
-ActiveRecord::ConnectionAdapters::RealTransaction.class_eval do
-  alias_method :commit_without_before_commit, :commit
-
-  def commit
-    call_before_commit_records if PubSubModelSync::Config.enable_rails4_before_commit
-    commit_without_before_commit
-  end
-
-  private
-
-  def call_before_commit_records
-    ite = records.uniq
-    ite.each do |record|
-      action = record.previous_changes.include?(:id) ? :create : :update
-      action = :destroy if record.destroyed?
-      callback_name = "ps_before_#{action}_commit".to_sym
-      record.send(callback_name) if record.respond_to?(callback_name)
-    end
-  end
-end