pub_sub_model_sync 0.5.4.1 → 0.5.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a06777a1f70a6efc94e7024b5b5c1f38d04f7eac8c9721a82fe84ffa042d15bb
-  data.tar.gz: f558c5116ec027c2b387dd04a16d03657d64f5fbefa244e9f075114cbd34ef21
+  metadata.gz: 14095a129de1a44dd2f51051e3ee768217eae9ddbaa69b2e9bb3979d857dde5a
+  data.tar.gz: eeff964335f21b2bc5db23048cf0a09cb8e4d385333f72cf431d9201e107737b
 SHA512:
-  metadata.gz: 41815c6d3390abcb71c3c4855e74e95ef7ead047e0e9315925eb8c1a56572f3458304d19bcdd38ceb51e203071f26a18071184fcbb766c47e5f617b2e0f530f4
-  data.tar.gz: 06f17b5f3ecfde95baf7c6465d371798a3257f2bbe86b288af4ab44250c8c8ff62faaa2597d4ddb520badf469a9558b2430e581d5d7c33bdfdd2d9e2ad83ae0a
+  metadata.gz: 3f85d85ac9f5de88af18848c8511e113a2359c5135b4dba8904e4bbc2b1f72b7be526c9fcdd221761202e9ef8d4630d84e08e9a5d8aa0c2753f98046a514fb70
+  data.tar.gz: 4788d133e2eaf6b302d34898d5f0d7002556edb357c907f280df137d686c71d7280503a763180f7fb61dece830c41f1263a7d569a868fad0082d13062e7caccd

data/CHANGELOG.md

@@ -1,7 +1,23 @@
 # Change Log
 
+# 0.5.8 (January 29, 2021)
+- fix: keep message ordering with google pubsub
+
+# 0.5.7.1 (January 26, 2021)
+- fix: does not call :on_error_processing when processing a message 
+
+# 0.5.7 (January 13, 2021)
+- feat: add method to preload sync listeners
+
+# 0.5.6 (January 12, 2021)
+- feat: add payload validation
+- feat: add method to rebuild payload
+
+# 0.5.5 (January 11, 2021)
+- feat: google-pub/sub receive messages in the same order they were delivered
+
 # 0.5.4.1 (January 8, 2021)
-- fix: make sequential message processor google-pub/sub
+- fix: google-pub/sub receive messages sequentially and not in parallel (default 5 threads).
 
 # 0.5.4 (January 8, 2021)
 - fix: exclude identifiers when syncing model

data/Gemfile

@@ -2,7 +2,7 @@ source "https://rubygems.org"
 
 gem 'rubocop', '~> 1.6.0', require: false
 gem 'bunny' # rabbit-mq
-gem 'google-cloud-pubsub' # google pub/sub
+gem 'google-cloud-pubsub', '> 2.0' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.4.1)
+    pub_sub_model_sync (0.5.8)
       rails
 
 GEM
@@ -78,49 +78,55 @@ GEM
     diff-lcs (1.3)
     digest-crc (0.5.1)
     erubi (1.10.0)
-    faraday (0.17.3)
+    faraday (1.1.0)
       multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    gapic-common (0.3.4)
+      google-protobuf (~> 3.12, >= 3.12.2)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.25)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-core (1.3.2)
+    google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
-    google-cloud-env (1.2.1)
-      faraday (~> 0.11)
-    google-cloud-pubsub (1.0.2)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.4.0)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.0.1)
+    google-cloud-pubsub (2.3.0)
       concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      grpc-google-iam-v1 (~> 0.6.9)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
-      rly (~> 0.2.3)
-    google-protobuf (3.11.4)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
-    googleauth (0.9.0)
-      faraday (~> 0.12)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.0)
+    google-cloud-pubsub-v1 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
+    google-protobuf (3.14.0-universal-darwin)
+    googleapis-common-protos (1.3.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    googleauth (0.14.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
-    grpc (1.27.0)
-      google-protobuf (~> 3.11)
+      signet (~> 0.14)
+    grpc (1.34.0-universal-darwin)
+      google-protobuf (~> 3.13)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.9)
-      googleapis-common-protos (>= 1.3.1, < 2.0)
-      grpc (~> 1.0)
+    grpc-google-iam-v1 (0.6.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos (>= 1.3.10, < 2.0)
+      grpc (~> 1.27)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    jwt (2.2.1)
+    jwt (2.2.2)
     loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -134,16 +140,16 @@ GEM
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.0)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    os (1.0.1)
+    os (1.1.1)
     parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.3)
+    public_suffix (4.0.6)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -177,7 +183,6 @@ GEM
     rake (13.0.1)
     regexp_parser (2.0.1)
     rexml (3.2.4)
-    rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -205,9 +210,10 @@ GEM
     ruby-kafka (1.0.0)
       digest-crc
     ruby-progressbar (1.10.1)
-    signet (0.11.0)
+    ruby2_keywords (0.0.2)
+    signet (0.14.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     sprockets (4.0.2)
@@ -235,7 +241,7 @@ DEPENDENCIES
   bundler
   bunny
   database_cleaner-active_record
-  google-cloud-pubsub
+  google-cloud-pubsub (> 2.0)
   pub_sub_model_sync!
   rake
   rspec

data/README.md

@@ -15,7 +15,7 @@ Add this line to your application's Gemfile:
 ```ruby
 gem 'pub_sub_model_sync'
 
-gem 'google-cloud-pubsub' # to use google pub/sub service
+gem 'google-cloud-pubsub', '>= 1.9' # to use google pub/sub service
 gem 'bunny' # to use rabbit-mq pub/sub service
 gem 'ruby-kafka' # to use apache kafka pub/sub service
 ```
@@ -59,10 +59,10 @@ And then execute: $ bundle install
     rake pub_sub_model_sync:start
     ```
     Note: Publishers do not need todo this    
-    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing without mentioned task (like rails console)
     ```ruby 
       # PubSubModelSync::Config.subscribers ==> []
-      Rails.application.try(:eager_load!)
+      PubSubModelSync::Runner.preload_listeners
       # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
     ``` 
 
@@ -308,9 +308,11 @@ config.debug = true
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)
-- Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
+- Add flag ```model.ps_process_payload``` to retrieve the payload used to process the pub/sub sync
 - Auto publish update only if payload has changed
-- On delete, payload must only be composed by ids 
+- On delete, payload must only be composed by ids
+- Feature to publish multiple message at a time with the ability to exclude similar messages by klass and action (use the last one)
+    PubSubModelSync::MessagePublisher.batch_publish({ same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })
 
 ## Q&A
 - Error "could not obtain a connection from the pool within 5.000 seconds"    
@@ -318,7 +320,28 @@ config.debug = true
   To fix the problem, edit config/database.yml and increase the quantity of ```pool: 10```
 - Google pubsub: How to process notifications parallely and not sequentially (default 1 thread)?    
   ```ruby  PubSubModelSync::ServiceGoogle::LISTEN_SETTINGS = { threads: { callback: qty_threads } } ```    
-  Note: by this way some notifications can be processed before others thus missing relationship errors can appear 
+  Note: by this way some notifications can be processed before others thus missing relationship errors can appear
+- How to retry failed syncs with sidekiq?
+  ```ruby
+    # lib/initializers/pub_sub_config.rb
+  
+    class PubSubRecovery
+      include Sidekiq::Worker
+      sidekiq_options queue: :pubsub, retry: 2, backtrace: true
+  
+      def perform(payload_data, action)
+        payload = PubSubModelSync::Payload.from_payload_data(payload_data)
+        payload.send(action)
+      end
+    end
+  
+    PubSubModelSync::Config.on_error_publish = lambda do |_e, data|
+      PubSubRecovery.perform_async(data[:payload].to_h, :publish!)
+    end
+    PubSubModelSync::Config.on_error_processing = lambda do |_e, data|
+      PubSubRecovery.perform_async(data[:payload].to_h, :process!)
+    end
+  ``` 
 
 ## Contributing
 

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -34,6 +34,14 @@ module PubSubModelSync
       def publish(*_args)
         true
       end
+
+      def publish_async(*_args)
+        yield(OpenStruct.new(succeeded?: true)) if block_given?
+      end
+
+      def enable_message_ordering!
+        true
+      end
     end
 
     def topic(*_args)

data/lib/pub_sub_model_sync/payload.rb

@@ -2,6 +2,7 @@
 
 module PubSubModelSync
   class Payload
+    class MissingInfo < StandardError; end
     attr_reader :data, :attributes, :headers
 
     # @param data (Hash: { any value }):
@@ -11,8 +12,10 @@ module PubSubModelSync
       @attributes = attributes
       @headers = headers
       build_headers
+      validate!
     end
 
+    # @return Hash: payload data
     def to_h
       { data: data, attributes: attributes, headers: headers }
     end
@@ -25,33 +28,52 @@ module PubSubModelSync
       attributes[:action]
     end
 
+    # Process payload data
+    #   (If error will raise exception and wont call on_error_processing callback)
     def process!
       process do |publisher|
         publisher.raise_error = true
       end
     end
 
+    # Process payload data
+    #   (If error will call on_error_processing callback)
     def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
       yield(publisher) if block_given?
       publisher.process
     end
 
+    # Publish payload to pubsub
+    #   (If error will raise exception and wont call on_error_publish callback)
     def publish!
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self, raise_error: true)
     end
 
+    # Publish payload to pubsub
+    #   (If error will call on_error_publish callback)
     def publish
       klass = PubSubModelSync::MessagePublisher
       klass.publish(self)
     end
 
+    # convert payload data into Payload
+    # @param data [Hash]: payload data (:data, :attributes, :headers)
+    def self.from_payload_data(data)
+      data = data.deep_symbolize_keys
+      new(data[:data], data[:attributes], data[:headers])
+    end
+
     private
 
     def build_headers
       headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
     end
+
+    def validate!
+      raise MissingInfo if !attributes[:klass] || !attributes[:action]
+    end
   end
 end

data/lib/pub_sub_model_sync/runner.rb

@@ -4,6 +4,7 @@ require 'active_support/core_ext/module'
 module PubSubModelSync
   class Runner
     class ShutDown < StandardError; end
+    delegate :preload_listeners, to: :class
     attr_accessor :connector
 
     def initialize
@@ -12,12 +13,17 @@ module PubSubModelSync
 
     def run
       trap_signals!
-      preload_framework!
+      preload_listeners
       start_listeners
     rescue ShutDown
       connector.stop
     end
 
+    def self.preload_listeners
+      Rails.application.try(:eager_load!) if defined?(Rails)
+      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
+    end
+
     private
 
     def start_listeners
@@ -31,10 +37,5 @@ module PubSubModelSync
       end
       %w[INT QUIT TERM].each { |signal| Signal.trap(signal, handler) }
     end
-
-    def preload_framework!
-      Rails.application.try(:eager_load!) if defined?(Rails)
-      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_base.rb

@@ -29,7 +29,7 @@ module PubSubModelSync
       if same_app_message?(payload)
         log("Skip message from same origin: #{[payload]}") if config.debug
       else
-        payload.process!
+        payload.process
       end
     rescue => e
       error = [payload, e.message, e.backtrace]

data/lib/pub_sub_model_sync/service_google.rb

@@ -8,13 +8,16 @@ end
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
     LISTEN_SETTINGS = { threads: { callback: 1 } }.freeze
+    TOPIC_SETTINGS = { async: { threads: { publish: 1, callback: 1 } } }.freeze
+    SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
     attr_accessor :service, :topic, :subscription, :subscriber
 
     def initialize
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
       @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name)
+               service.create_topic(config.topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
     end
 
     def listen_messages
@@ -29,7 +32,9 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
+      topic.publish_async(payload.to_json, message_headers) do |res|
+        raise 'Failed to publish the message.' unless res.succeeded?
+      end
     end
 
     def stop
@@ -39,9 +44,13 @@ module PubSubModelSync
 
     private
 
+    def message_headers
+      { SERVICE_KEY => true, ordering_key: Time.current.to_i }.merge(PUBLISH_SETTINGS)
+    end
+
     def subscribe_to_topic
       topic.subscription(config.subscription_key) ||
-        topic.subscribe(config.subscription_key)
+        topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)
     end
 
     def process_message(received_message)

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.4.1'
+  VERSION = '0.5.8'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.4.1
+  version: 0.5.8
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-08 00:00:00.000000000 Z
+date: 2021-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails