pub_sub_model_sync 0.5.7.1 → 0.5.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd5c9fd9bd9c35d62c1eeea4fc91024a5c14f136a757c82bde849501e0445c9d
-  data.tar.gz: 20ea1422d130daa72863743cae0716ba01455e445ee96a9410ec050f0b7f5e52
+  metadata.gz: 585431eec8ef5f6f9d4b0ccea9069d03ea2849d94f7a8f43464f588da8cfae6c
+  data.tar.gz: 4ebf42b8ca72c0bf150f3c27a9ca9b791532954deb1663c571927324be952589
 SHA512:
-  metadata.gz: 4c1c1afba624ef824b269ab7e6aeb112b40bc932b9125ffb90b46b03f7aea96dfc3359fd3ba780b74f9145d37b7fc01be279c8dcf2ecbadce23ac4faad821441
-  data.tar.gz: c908efda5d92bcf8b262586feb22cadc884c24033a43318d22bdd16fb90036d139994fefc02a7b33335dd38f176cb93096e17aca4cd519d4ed8222d911848f69
+  metadata.gz: 1f0c786ff6de119c0add336c0182158f4c5ab3b9b74d2f9e3f180b3753ac288ecc8e2c6c88149ad82a627691c924f049eeb5e02c39280ec7ce3d04831b99dd18
+  data.tar.gz: f7ae9286d95af747dd8bd5e21c5f4f87e8aec469077663782103738f7ae9d07fa39b4884f5741e572f17b57a1ce6c358fe8ded7334451aa703559b7367218653

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Change Log
 
+# 0.5.9.1 (February 10, 2021)
+- feat: move :key into headers
+
+# 0.5.9 (February 10, 2021)
+- feat: reformat :publish and :process methods to include non silence methods
+- feat: add notification key to payloads (can be used for caching strategies)
+
+# 0.5.8.2 (February 05, 2021)
+- fix: restore google pubsub topic settings
+
+# 0.5.8.1 (February 05, 2021)
+- fix: keep message ordering with google pubsub
+
+# 0.5.8 (January 29, 2021)
+- fix: keep message ordering with google pubsub
+
 # 0.5.7.1 (January 26, 2021)
 - fix: does not call :on_error_processing when processing a message 
 

data/Gemfile

@@ -2,7 +2,7 @@ source "https://rubygems.org"
 
 gem 'rubocop', '~> 1.6.0', require: false
 gem 'bunny' # rabbit-mq
-gem 'google-cloud-pubsub' # google pub/sub
+gem 'google-cloud-pubsub', '> 2.0' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.7.1)
+    pub_sub_model_sync (0.5.9.1)
       rails
 
 GEM
@@ -78,49 +78,55 @@ GEM
     diff-lcs (1.3)
     digest-crc (0.5.1)
     erubi (1.10.0)
-    faraday (0.17.3)
+    faraday (1.1.0)
       multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    gapic-common (0.3.4)
+      google-protobuf (~> 3.12, >= 3.12.2)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.25)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-core (1.3.2)
+    google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
-    google-cloud-env (1.2.1)
-      faraday (~> 0.11)
-    google-cloud-pubsub (1.0.2)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.4.0)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.0.1)
+    google-cloud-pubsub (2.3.0)
       concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      grpc-google-iam-v1 (~> 0.6.9)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
-      rly (~> 0.2.3)
-    google-protobuf (3.11.4)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
-    googleauth (0.9.0)
-      faraday (~> 0.12)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.0)
+    google-cloud-pubsub-v1 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
+    google-protobuf (3.14.0-universal-darwin)
+    googleapis-common-protos (1.3.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    googleauth (0.14.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
-    grpc (1.27.0)
-      google-protobuf (~> 3.11)
+      signet (~> 0.14)
+    grpc (1.34.0-universal-darwin)
+      google-protobuf (~> 3.13)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.9)
-      googleapis-common-protos (>= 1.3.1, < 2.0)
-      grpc (~> 1.0)
+    grpc-google-iam-v1 (0.6.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos (>= 1.3.10, < 2.0)
+      grpc (~> 1.27)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    jwt (2.2.1)
+    jwt (2.2.2)
     loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -134,16 +140,16 @@ GEM
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.0)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    os (1.0.1)
+    os (1.1.1)
     parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.3)
+    public_suffix (4.0.6)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -177,7 +183,6 @@ GEM
     rake (13.0.1)
     regexp_parser (2.0.1)
     rexml (3.2.4)
-    rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -205,9 +210,10 @@ GEM
     ruby-kafka (1.0.0)
       digest-crc
     ruby-progressbar (1.10.1)
-    signet (0.11.0)
+    ruby2_keywords (0.0.2)
+    signet (0.14.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     sprockets (4.0.2)
@@ -235,7 +241,7 @@ DEPENDENCIES
   bundler
   bunny
   database_cleaner-active_record
-  google-cloud-pubsub
+  google-cloud-pubsub (> 2.0)
   pub_sub_model_sync!
   rake
   rspec

data/README.md

@@ -15,7 +15,7 @@ Add this line to your application's Gemfile:
 ```ruby
 gem 'pub_sub_model_sync'
 
-gem 'google-cloud-pubsub' # to use google pub/sub service
+gem 'google-cloud-pubsub', '>= 1.9' # to use google pub/sub service
 gem 'bunny' # to use rabbit-mq pub/sub service
 gem 'ruby-kafka' # to use apache kafka pub/sub service
 ```
@@ -50,7 +50,19 @@ And then execute: $ bundle install
     PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], { logger: Rails.logger }]
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
-    See details here: https://github.com/zendesk/ruby-kafka    
+    See details here: https://github.com/zendesk/ruby-kafka
+    
+    Kafka Confluence example:
+    ```ruby
+      kafka_settings = {
+        sasl_plain_username: '...',
+        sasl_plain_password: '...',
+        ssl_ca_certs_from_system: true,
+        logger: Rails.logger, client_id: 'my-app-name'
+      }
+      PubSubModelSync::Config.kafka_connection = [['...confluent.cloud:9092'], kafka_settings]  
+    ```    
+    Note: You need to create the topic manually on Kafka Confluence
 
 - Add publishers/subscribers to your models (See examples below)
 
@@ -308,10 +320,19 @@ config.debug = true
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)
-- Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
+- Add flag ```model.ps_process_payload``` to retrieve the payload used to process the pub/sub sync
 - Auto publish update only if payload has changed
 - On delete, payload must only be composed by ids
-- Change notifications into messages 
+- Feature to publish multiple message at a time with the ability to exclude similar messages by klass and action (use the last one)
+    PubSubModelSync::MessagePublisher.batch_publish({ same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })
+- Add DB table to use as a shield to skip publishing similar notifications or publish partial notifications (similar idea when processing notif)
+- add callback: on_message_received(payload)
+- Sometimes the listener service stops listening, debug idea:
+```subscriber.on_error do
+     log('error gogooogle')
+   end
+   loop{ log(['loooooop:', {l: subscriber.last_error, stopped: subscriber.stopped?, started: subscriber.started?}]); sleep 2 }
+```
 
 ## Q&A
 - Error "could not obtain a connection from the pool within 5.000 seconds"    

data/lib/pub_sub_model_sync/message_processor.rb

@@ -2,7 +2,7 @@
 
 module PubSubModelSync
   class MessageProcessor < PubSubModelSync::Base
-    attr_accessor :payload, :raise_error
+    attr_accessor :payload
 
     # @param payload (Payload): payload to be delivered
     # @Deprecated: def initialize(data, klass, action)
@@ -15,10 +15,16 @@ module PubSubModelSync
       @payload = PubSubModelSync::Payload.new(payload, { klass: klass, action: action })
     end
 
-    def process
+    def process!
       filter_subscribers.each(&method(:run_subscriber))
     end
 
+    def process
+      process!
+    rescue => e
+      notify_error(e)
+    end
+
     private
 
     def run_subscriber(subscriber)
@@ -29,8 +35,6 @@ module PubSubModelSync
         res = config.on_success_processing.call(payload, { subscriber: subscriber })
         log "processed message with: #{payload.inspect}" if res != :skip_log
       end
-    rescue => e
-      raise_error ? raise : print_subscriber_error(e, subscriber)
     end
 
     def processable?(subscriber)
@@ -40,9 +44,9 @@ module PubSubModelSync
     end
 
     # @param error (Error)
-    def print_subscriber_error(error, subscriber)
+    def notify_error(error)
       info = [payload, error.message, error.backtrace]
-      res = config.on_error_processing.call(error, { payload: payload, subscriber: subscriber })
+      res = config.on_error_processing.call(error, { payload: payload })
       log("Error processing message: #{info}", :error) if res != :skip_log
     end
 

data/lib/pub_sub_model_sync/message_publisher.rb

@@ -7,14 +7,20 @@ module PubSubModelSync
         @connector ||= PubSubModelSync::Connector.new
       end
 
+      # Publishes any value to pubsub
+      # @param klass (String): Class name
+      # @param data (Hash): Data to be delivered
+      # @param action (:symbol): action name
       def publish_data(klass, data, action)
-        payload = PubSubModelSync::Payload.new(data, { klass: klass.to_s, action: action.to_sym })
+        attrs = { klass: klass.to_s, action: action.to_sym }
+        payload = PubSubModelSync::Payload.new(data, attrs)
         publish(payload)
       end
 
-      # @param model: ActiveRecord model
-      # @param action: (Sym) Action name
-      # @param publisher: (Publisher, optional) Publisher to be used
+      # Publishes model info to pubsub
+      # @param model (ActiveRecord model)
+      # @param action (Sym): Action name
+      # @param publisher (Publisher, optional): Publisher to be used
       def publish_model(model, action, publisher = nil)
         return if model.ps_skip_sync?(action)
 
@@ -27,7 +33,10 @@ module PubSubModelSync
         model.ps_after_sync(action, payload.data)
       end
 
-      def publish(payload, raise_error: false)
+      # Publishes payload to pubsub
+      # @attr payload (PubSubModelSync::Payload)
+      # Raises error if exist
+      def publish!(payload)
         if config.on_before_publish.call(payload) == :cancel
           log("Publish message cancelled: #{payload}") if config.debug
           return
@@ -36,8 +45,14 @@ module PubSubModelSync
         log("Publishing message: #{[payload]}")
         connector.publish(payload)
         config.on_after_publish.call(payload)
+      end
+
+      # Similar to :publish! method
+      # Notifies error via :on_error_publish instead of raising error
+      def publish(payload)
+        publish!(payload)
       rescue => e
-        raise_error ? raise : notify_error(e, payload)
+        notify_error(e, payload)
       end
 
       private

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -34,6 +34,14 @@ module PubSubModelSync
       def publish(*_args)
         true
       end
+
+      def publish_async(*_args)
+        yield(OpenStruct.new(succeeded?: true)) if block_given?
+      end
+
+      def enable_message_ordering!
+        true
+      end
     end
 
     def topic(*_args)

data/lib/pub_sub_model_sync/payload.rb

@@ -6,7 +6,8 @@ module PubSubModelSync
     attr_reader :data, :attributes, :headers
 
     # @param data (Hash: { any value }):
-    # @param attributes (Hash: { klass: string, action: :sym }):
+    # @param attributes (Hash: { klass*: string, action*: :sym }):
+    # @param headers (Hash: { key?: string, ...any_key?: anything }):
     def initialize(data, attributes, headers = {})
       @data = data
       @attributes = attributes
@@ -31,16 +32,14 @@ module PubSubModelSync
     # Process payload data
     #   (If error will raise exception and wont call on_error_processing callback)
     def process!
-      process do |publisher|
-        publisher.raise_error = true
-      end
+      publisher = PubSubModelSync::MessageProcessor.new(self)
+      publisher.process!
     end
 
     # Process payload data
     #   (If error will call on_error_processing callback)
     def process
       publisher = PubSubModelSync::MessageProcessor.new(self)
-      yield(publisher) if block_given?
       publisher.process
     end
 
@@ -48,7 +47,7 @@ module PubSubModelSync
     #   (If error will raise exception and wont call on_error_publish callback)
     def publish!
       klass = PubSubModelSync::MessagePublisher
-      klass.publish(self, raise_error: true)
+      klass.publish!(self)
     end
 
     # Publish payload to pubsub
@@ -70,6 +69,7 @@ module PubSubModelSync
     def build_headers
       headers[:uuid] ||= SecureRandom.uuid
       headers[:app_key] ||= PubSubModelSync::Config.subscription_key
+      headers[:key] ||= [klass.to_s, action].join('/')
     end
 
     def validate!

data/lib/pub_sub_model_sync/publisher.rb

@@ -11,8 +11,10 @@ module PubSubModelSync
       @as_klass = as_klass || klass
     end
 
+    # Builds the payload with model information defined for :action (:create|:update|:destroy)
     def payload(model, action)
-      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action))
+      headers = { key: [model.class.name, action, model.id].join('/') }
+      PubSubModelSync::Payload.new(payload_data(model), payload_attrs(model, action), headers)
     end
 
     private
@@ -29,7 +31,10 @@ module PubSubModelSync
     end
 
     def payload_attrs(model, action)
-      { klass: (as_klass || model.class.name).to_s, action: action.to_sym }
+      {
+        klass: (as_klass || model.class.name).to_s,
+        action: action.to_sym
+      }
     end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,7 +7,8 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 } }.freeze
+    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true }.freeze
+    TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
     attr_accessor :service, :topic, :subscription, :subscriber
 
@@ -15,7 +16,8 @@ module PubSubModelSync
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
       @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name)
+               service.create_topic(config.topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
     end
 
     def listen_messages
@@ -30,7 +32,9 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
+      topic.publish_async(payload.to_json, message_headers) do |res|
+        raise 'Failed to publish the message.' unless res.succeeded?
+      end
     end
 
     def stop
@@ -40,6 +44,10 @@ module PubSubModelSync
 
     private
 
+    def message_headers
+      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    end
+
     def subscribe_to_topic
       topic.subscription(config.subscription_key) ||
         topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.7.1'
+  VERSION = '0.5.9.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.7.1
+  version: 0.5.9.1
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-27 00:00:00.000000000 Z
+date: 2021-02-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails