pub_sub_model_sync 1.7.1 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4dbf0fae81e981722c2a5dc1023dc3db9c369cfcdc5558ef727e59b1d6c87a8a
-  data.tar.gz: 340032eb2b9d47de47aa4e57c3db389c5571280df75a92fee96b52de2cc8c19e
+  metadata.gz: 450a4434b9a66faafa91e958b2c6aa42b4d9d1732cc591609e81f4d9f3d581e6
+  data.tar.gz: d9577d4757bd712fd8e8e8b8289dbce83ab57b53b92fb0d1f39f903fcca53a04
 SHA512:
-  metadata.gz: f65eac76d6e247c89d863f75476539bf75324b12c510bdce2eaedc9667168381df82a39f3a2ee24c6da1c39b447159981512d9c744233b835c7000a95b514ac1
-  data.tar.gz: 1e549a6a809ecb162eaf6c4101a9a379f2ea2a1cccacb975854de02c1ddf928a54a05b9c432cdd26ac31863fcf5cffbefaa3baf0753f4301054fc509167d3316
+  metadata.gz: 39c90ccb110f229296fcfca67184b7b37bfe1a7c533fbbfe6b30c58330e4b46041e7775b5c7a3938c59412368dbce8cffe4684fc4ff2c15a46b56319026072cb
+  data.tar.gz: ae4ec687e8b8e1e19e89bf223b5abb12bcac57481ab2e1b5cfef2121433925f79f7c2a557709a51e8ddea8a52866db90d68f7c3cc1524c9548ddace85d1d62d3

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.7.0)
+    pub_sub_model_sync (1.8.0)
       rails
 
 GEM

data/README.md

@@ -83,7 +83,9 @@ And then execute: $ bundle install
     PubSubModelSync::Config.topic_name = 'sample-topic'
     PubSubModelSync::Config.subscription_name = 'my-app3'
     ```
-    See details here: https://github.com/zendesk/ruby-kafka
+    See details here: https://github.com/zendesk/ruby-kafka    
+
+    *Important: The `topic_name` must be same for all applications, so that, the apps connect to the same topic*
 
 - Add publishers/subscribers to your models (See examples below)
 
@@ -382,8 +384,7 @@ Note: To reduce Payload size, some header info are not delivered (Enable debug m
   
   - Manual transactions   
     `PubSubModelSync::MessagePublisher::transaction(key, headers: { target_app_key: 'my_other_app_key' } &block)`
-    - `key` (String|nil) Key used as the ordering_key for all inner notifications (When nil, will use `ordering_key` of the first notification)  
-    - `max_buffer:` (Integer, default: `PubSubModelSync::Config.transactions_max_buffer`) Transaction buffer size (DEPRECATED).     
+    - `key` (String|nil) Key used as the ordering_key for all inner notifications (When nil, will use `ordering_key` of the first notification)
     - `headers:` (Hash) Header settings to be added to each Payload's header inside this transaction     
     Sample:
     ```ruby
@@ -542,7 +543,9 @@ config.debug = true
 - ```.on_error_publish = ->(exception, {payload:}) { payload.delay(...).retry_publish! }```
     (Proc) => called when failed publishing a notification (delayed_job or similar can be used for retrying)
 - ```.skip_cache = false```
-    (true/false*) => Allow to skip payload optimization (cache settings)
+  (true/false*) => Allow to skip payload optimization (cache settings)
+- ```.sync_mode = true```
+    (true/false*) => If `true`, the messages are delivered synchronously, else, they are delivered asynchronously (Currently, only GooglePubsub supports it). Also it can be enabled via env var: `PUBSUB_MODEL_SYNC_MODE=true`
 - ```.transactions_max_buffer = 1``` (Integer, default 1) Controls the maximum quantity of notifications to be enqueued to the transaction-buffer before delivering them and thus adds the ability to rollback notifications if the transaction fails.        
     Once this quantity of notifications is reached, then all notifications of the current transaction will immediately be delivered (can be customized per transaction).    
     Note: There is no way to rollback delivered notifications if current transaction fails later.      
@@ -562,6 +565,7 @@ config.debug = true
 - Add subscription liveness checker using thread without db connection to check periodically pending notifications from google pubsub
 - Unify .stop() and 'Listener stopped' 
 - TODO: Publish new version 1.2.1 (improve logs)
+- Enable `async` mode for rabbitMQ and Kafka
 
 ## **Q&A**
 - I'm getting error "could not obtain a connection from the pool within 5.000 seconds"... what does this mean?

data/lib/pub_sub_model_sync/config.rb

@@ -10,6 +10,7 @@ module PubSubModelSync
     cattr_accessor(:logger) { Rails.logger }
     cattr_accessor(:transactions_max_buffer) { 1 }
     cattr_accessor(:skip_cache) { false }
+    cattr_accessor(:sync_mode) { ENV['PUBSUB_MODEL_SYNC_MODE'] == 'true' }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }

data/lib/pub_sub_model_sync/message_processor.rb

@@ -17,8 +17,7 @@ module PubSubModelSync
 
     def process!
       subscribers = filter_subscribers
-      payload_info = { klass: payload.klass, action: payload.action, mode: payload.mode }
-      log("No subscribers found for #{payload_info}", :warn) if config.debug && subscribers.empty?
+      log("No subscribers found for #{payload.uuid}", :warn) if config.debug && subscribers.empty?
       subscribers.each(&method(:run_subscriber))
     rescue => e
       print_error(e)
@@ -38,7 +37,7 @@ module PubSubModelSync
       processor = PubSubModelSync::RunSubscriber.new(subscriber, payload)
       return unless processable?(subscriber)
 
-      log("Processing message #{[subscriber, payload]}...") if config.debug
+      log("Processing message #{[subscriber, payload.uuid]}...") if config.debug
       processor.call
       res = config.on_success_processing.call(payload, { subscriber: subscriber })
       log "processed message with: #{payload.inspect}" if res != :skip_log
@@ -48,7 +47,7 @@ module PubSubModelSync
 
     def processable?(subscriber)
       cancel = config.on_before_processing.call(payload, { subscriber: subscriber }) == :cancel
-      log("process message cancelled: #{payload}") if cancel && config.debug
+      log("process message cancelled via on_before_processing: #{payload.uuid}") if cancel && config.debug
       !cancel
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -94,7 +94,7 @@ module PubSubModelSync
       end
 
       def connector_publish(payload)
-        log("Publishing message #{[payload]}...") if config.debug
+        log("Publishing message #{[payload.uuid]}...") if config.debug
         connector.publish(payload)
         log("Published message: #{[payload]}")
         config.on_after_publish.call(payload)

data/lib/pub_sub_model_sync/payload.rb

@@ -55,6 +55,14 @@ module PubSubModelSync
       (info[:mode] || :model).to_sym
     end
 
+    def uuid
+      headers[:uuid]
+    end
+
+    def ordering_key
+      headers[:ordering_key]
+    end
+
     # Process payload data
     #   (If error will raise exception and wont call on_error_processing callback)
     def process!
@@ -88,10 +96,7 @@ module PubSubModelSync
 
     # @param attr_keys (Array<Symbol>) List of attributes to be excluded from payload
     def exclude_data_attrs(attr_keys)
-      orig_data = data.clone
-      headers[:excluded_attr_keys] = attr_keys.join(',')
       @data = data.except(*attr_keys)
-      Config.log("Empty payload after payload optimization (original data: #{[self, orig_data]})") if @data == []
     end
 
     # Attributes to always be delivered after cache optimization

data/lib/pub_sub_model_sync/payload_cache_optimizer.rb

@@ -32,7 +32,7 @@ module PubSubModelSync
 
     def cache_disabled?
       res = config.skip_cache || Rails.cache.nil?
-      log("Skipping cache, it was disabled: #{[payload]}") if res && debug?
+      log("Skipping cache, it was disabled: #{[payload.uuid]}") if res && debug?
       res
     end
 
@@ -44,7 +44,7 @@ module PubSubModelSync
       changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
       required_keys = payload.cache_settings[:required].map(&:to_sym)
       invalid_keys = payload.data.keys - (changed_keys + required_keys)
-      log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
+      log("Excluding non changed attributes: #{invalid_keys} from: #{payload.uuid}") if debug?
       payload.exclude_data_attrs(invalid_keys)
     end
   end

data/lib/pub_sub_model_sync/run_subscriber.rb

@@ -40,11 +40,11 @@ module PubSubModelSync
       call_action(model)
     end
 
-    def ensure_sync(object)
+    def ensure_sync(object) # rubocop:disable Metrics/AbcSize
       res = true
       res = false if settings[:if] && !parse_condition(settings[:if], object)
       res = false if settings[:unless] && parse_condition(settings[:unless], object)
-      log("Cancelled save sync by subscriber condition : #{[payload]}") if !res && debug?
+      log("Cancelled save sync by subscriber condition : #{[payload.uuid]}") if !res && debug?
       res
     end
 

data/lib/pub_sub_model_sync/service_base.rb

@@ -42,7 +42,7 @@ module PubSubModelSync
     # @return [Payload,Nil]
     def decode_payload(payload_info)
       payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
-      log("Received message: #{[payload]}") if config.debug
+      log("Received message: #{payload.uuid}") if config.debug
       payload
     rescue => e
       error_payload = [payload_info, e.message, e.backtrace]
@@ -54,7 +54,7 @@ module PubSubModelSync
     def same_app_message?(payload)
       key = payload.headers[:app_key]
       res = key && key == config.subscription_key
-      log("Skipping message from same origin: #{[payload]}") if res && config.debug
+      log("Skipping message from same origin: #{payload.uuid}") if res && config.debug
       res
     end
 

data/lib/pub_sub_model_sync/service_google.rb

@@ -54,17 +54,25 @@ module PubSubModelSync
 
     def publish_to_topic(topic, payload)
       retries ||= 0
-      topic.publish_async(encode_payload(payload), message_headers(payload)) do |res|
-        raise StandardError, "Failed to publish the message. #{res.error}" unless res.succeeded?
-      end
+      config.sync_mode ? topic.publish(*message_params(payload)) : publish_async(topic, payload)
     rescue Google::Cloud::PubSub::OrderingKeyError => e
       raise if (retries += 1) > 1
 
-      log("Resuming ordering_key and retrying OrderingKeyError for #{payload.headers[:uuid]}: #{e.message}")
-      topic.resume_publish(message_headers(payload)[:ordering_key])
+      log("Resuming ordering_key and retrying OrderingKeyError for #{payload.uuid}: #{e.message}")
+      topic.resume_publish(payload.ordering_key)
       retry
     end
 
+    def publish_async(topic, payload)
+      topic.publish_async(*message_params(payload)) do |result|
+        log "Published message: #{payload.uuid} (via async)" if result.succeeded? && config.debug
+        unless result.succeeded?
+          log("Error publishing: #{[payload, result.error]} (via async)", :error)
+          config.on_error_publish.call(StandardError.new(result.error), { payload: payload })
+        end
+      end
+    end
+
     # @param only_publish (Boolean): if false is used to listen and publish messages
     # @return (Topic): returns created or loaded topic
     def init_topic(topic_name, only_publish: false)
@@ -79,11 +87,12 @@ module PubSubModelSync
     end
 
     # @param payload (PubSubModelSync::Payload)
-    def message_headers(payload)
-      {
-        SERVICE_KEY => true,
-        ordering_key: payload.headers[:ordering_key]
-      }.merge(PUBLISH_SETTINGS)
+    # @return [Array]
+    def message_params(payload)
+      [
+        encode_payload(payload),
+        { ordering_key: payload.ordering_key, SERVICE_KEY => true }.merge(PUBLISH_SETTINGS)
+      ]
     end
 
     # @return [Array<Subscriber>]

data/lib/pub_sub_model_sync/transaction.rb

@@ -18,13 +18,9 @@ module PubSubModelSync
     end
 
     # @param payload (Payload)
+    # TODO: remove buffer (already managed by pubsub services when running with config.async = true)
     def add_payload(payload)
       payloads << payload
-      print_log = config.debug && max_buffer > 1
-      log("Payload added to current transaction: #{payload.inspect}") if print_log
-      return unless payloads.count >= max_buffer
-
-      log("Payloads buffer was filled, delivering current payloads: #{payloads.count}") if print_log
       deliver_payloads
     end
 
@@ -44,7 +40,6 @@ module PubSubModelSync
     end
 
     def rollback
-      log("Rollback #{payloads.count} notifications", :warn) if children.any? && debug?
       self.children = []
       root&.rollback
       clean_publisher
@@ -68,8 +63,6 @@ module PubSubModelSync
 
     def deliver_payload(payload)
       PUBLISHER_KLASS.connector_publish(payload)
-    rescue => e
-      PUBLISHER_KLASS.send(:notify_error, e, payload)
     end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.7.1'
+  VERSION = '1.9.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.7.1
+  version: 1.9.0
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-12-29 00:00:00.000000000 Z
+date: 2023-01-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails