pub_sub_model_sync 0.5.4 → 0.5.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 57e8d23c9e514734a8a45664f641378bbb46e8de5315c536a0f692efcbe29dd2
-  data.tar.gz: d0f290dac2b0563ffe5412f85e1a6a6e340e281049755b26dc732a3661501b92
+  metadata.gz: cd5c9fd9bd9c35d62c1eeea4fc91024a5c14f136a757c82bde849501e0445c9d
+  data.tar.gz: 20ea1422d130daa72863743cae0716ba01455e445ee96a9410ec050f0b7f5e52
 SHA512:
-  metadata.gz: 351901f98e6c806e7a4efe20abdac65de5bf8f716dc57e1889302eab02d33b3f2f4d449cd0d351fd5ae882d16abe79a77791bea8ad956c52cda47829319eb858
-  data.tar.gz: e906d601a8c37b999a8b6d550e605e9c04b67db692cec4b60220ae286bef9adde7132d8614cee4ca7dab9ef43538ae2baede3f07e402c4c21a189b79a9ba6873
+  metadata.gz: 4c1c1afba624ef824b269ab7e6aeb112b40bc932b9125ffb90b46b03f7aea96dfc3359fd3ba780b74f9145d37b7fc01be279c8dcf2ecbadce23ac4faad821441
+  data.tar.gz: c908efda5d92bcf8b262586feb22cadc884c24033a43318d22bdd16fb90036d139994fefc02a7b33335dd38f176cb93096e17aca4cd519d4ed8222d911848f69

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Change Log
 
+# 0.5.7.1 (January 26, 2021)
+- fix: does not call :on_error_processing when processing a message 
+
+# 0.5.7 (January 13, 2021)
+- feat: add method to preload sync listeners
+
+# 0.5.6 (January 12, 2021)
+- feat: add payload validation
+- feat: add method to rebuild payload
+
+# 0.5.5 (January 11, 2021)
+- feat: google-pub/sub receive messages in the same order they were delivered
+
+# 0.5.4.1 (January 8, 2021)
+- fix: google-pub/sub receive messages sequentially and not in parallel (default 5 threads).
+
 # 0.5.4 (January 8, 2021)
 - fix: exclude identifiers when syncing model
 - feat: callbacks support for future extra params

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.4)
+    pub_sub_model_sync (0.5.7.1)
       rails
 
 GEM

data/README.md

@@ -59,10 +59,10 @@ And then execute: $ bundle install
     rake pub_sub_model_sync:start
     ```
     Note: Publishers do not need todo this    
-    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing without mentioned task (like rails console)
     ```ruby 
       # PubSubModelSync::Config.subscribers ==> []
-      Rails.application.try(:eager_load!)
+      PubSubModelSync::Runner.preload_listeners
       # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
     ``` 
 
@@ -310,7 +310,37 @@ config.debug = true
 - Add alias attributes when subscribing (similar to publisher)
 - Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
 - Auto publish update only if payload has changed
-- On delete, payload must only be composed by ids 
+- On delete, payload must only be composed by ids
+- Change notifications into messages 
+
+## Q&A
+- Error "could not obtain a connection from the pool within 5.000 seconds"    
+  This problem occurs because pub/sub dependencies (kafka, google-pubsub, rabbitmq) use many threads to perform notifications where the qty of threads is greater than qty of DB pools ([Google pubsub info](https://github.com/googleapis/google-cloud-ruby/blob/master/google-cloud-pubsub/lib/google/cloud/pubsub/subscription.rb#L888))     
+  To fix the problem, edit config/database.yml and increase the quantity of ```pool: 10```
+- Google pubsub: How to process notifications parallely and not sequentially (default 1 thread)?    
+  ```ruby  PubSubModelSync::ServiceGoogle::LISTEN_SETTINGS = { threads: { callback: qty_threads } } ```    
+  Note: by this way some notifications can be processed before others thus missing relationship errors can appear
+- How to retry failed syncs with sidekiq?
+  ```ruby
+    # lib/initializers/pub_sub_config.rb
+  
+    class PubSubRecovery
+      include Sidekiq::Worker
+      sidekiq_options queue: :pubsub, retry: 2, backtrace: true
+  
+      def perform(payload_data, action)
+        payload = PubSubModelSync::Payload.from_payload_data(payload_data)
+        payload.send(action)
+      end
+    end
+  
+    PubSubModelSync::Config.on_error_publish = lambda do |_e, data|
+      PubSubRecovery.perform_async(data[:payload].to_h, :publish!)
+    end
+    PubSubModelSync::Config.on_error_processing = lambda do |_e, data|
+      PubSubRecovery.perform_async(data[:payload].to_h, :process!)
+    end
+  ``` 
 
 ## Contributing
 

data/lib/pub_sub_model_sync/payload.rb

@@ -2,6 +2,7 @@
 
 module PubSubModelSync
   class Payload
+    class MissingInfo < StandardError; end
     attr_reader :data, :attributes, :headers
 
     # @param data (Hash: { any value }):
@@ -11,8 +12,10 @@ module PubSubModelSync
       @attributes = attributes
       @headers = headers
       build_headers
+      validate!
     end
 
+    # @return Hash: payload data
     def to_h
       { data: data, attributes: attributes, headers: headers }
     end
@@ -25,33 +28,52 @@ module PubSubModelSync
       attributes[:action]
     end
 
+    # Process payload data
+    #   (If error will raise exception and wont call on_error_processing callback)
     def process!
       process do |publisher|
         publisher.raise_error = true
       end
     end
 
+    # Process payload data
+    #   (If error will call on_error_processing callback)
     def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
       yield(publisher) if block_given?
       publisher.process
     end
 
+    # Publish payload to pubsub
+    #   (If error will raise exception and wont call on_error_publish callback)
     def publish!
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self, raise_error: true)
     end
 
+    # Publish payload to pubsub
+    #   (If error will call on_error_publish callback)
     def publish
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self)
     end
 
+    # convert payload data into Payload
+    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    def self.from_payload_data(data)
+      data = data.deep_symbolize_keys
+      new(data[:data], data[:attributes], data[:headers])
+    end
+
     private
 
     def build_headers
       headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
     end
+
+    def validate!
+      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+    end
   end
 end

data/lib/pub_sub_model_sync/runner.rb

@@ -4,6 +4,7 @@ require 'active_support/core_ext/module'
 module PubSubModelSync
   class Runner
     class ShutDown < StandardError; end
+    delegate :preload_listeners, to: :class
     attr_accessor :connector
 
     def initialize
@@ -12,12 +13,17 @@ module PubSubModelSync
 
     def run
       trap_signals!
-      preload_framework!
+      preload_listeners
       start_listeners
     rescue ShutDown
       connector.stop
     end
 
+    def self.preload_listeners
+      Rails.application.try(:eager_load!) if defined?(Rails)
+      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
+    end
+
     private
 
     def start_listeners
@@ -31,10 +37,5 @@ module PubSubModelSync
       end
       %w[INT QUIT TERM].each { |signal| Signal.trap(signal, handler) }
     end
-
-    def preload_framework!
-      Rails.application.try(:eager_load!) if defined?(Rails)
-      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_base.rb

@@ -29,7 +29,7 @@ module PubSubModelSync
       if same_app_message?(payload)
         log("Skip message from same origin: #{[payload]}") if config.debug
       else
-        payload.process!
+        payload.process
       end
     rescue => e
       error = [payload, e.message, e.backtrace]

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,6 +7,8 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
+    LISTEN_SETTINGS = { threads: { callback: 1 } }.freeze
+    SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
     attr_accessor :service, :topic, :subscription, :subscriber
 
     def initialize
@@ -40,7 +42,7 @@ module PubSubModelSync
 
     def subscribe_to_topic
       topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key)
+        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.4'
+  VERSION = '0.5.7.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.4
+  version: 0.5.7.1
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-08 00:00:00.000000000 Z
+date: 2021-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails