pub_sub_model_sync 0.5.6 → 0.5.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebc62742739be4d3291fc1551a8f53bf57d815f7b0ce3a9622e4c06a025e4fab
-  data.tar.gz: ec84b83f7b2e176a2030415674b6c09859e40bbd9069fa40ff51041d24de0d4e
+  metadata.gz: 7ae06ef31975d868894ce10558fe100df08710d50ba356318c57e71f5e3e687b
+  data.tar.gz: 3b218c8f1f973a0a4db79d747dd05f0d191dab9a684d15d88b84c7de261eff6f
 SHA512:
-  metadata.gz: b351a1210c3b747de830e4ed67c527625d7941967e71332189000de5b81cb4286f906d8ab943595c7f7ab4cf7c2238c5458b2ea87620a11705877889552dcfcf
-  data.tar.gz: 6d282613d88dfa53460bab1425c10553e89837a31790a49bbbd27f451de39f777b56d99424ef81e046eb459ba5d6b97aea92d6b812830579e9a2996e67c5dc6f
+  metadata.gz: aadd4d7095dc9114fd1f645bbe8c6105c0ffc0fff44cd3bcbf10b7d170ccc8c45257d07b2375b88e413ddc0a6b2bcf54293958bde10603dd7485b4adf1bbd2da
+  data.tar.gz: c4fd4aff648b7a9c504e2a7d81982aa1238ef47be387198e11d9c9f3f0325fb719faaa70a2ce93693526dae8ee578e03d4e2a2c62a5ceb24ee59f0d172444cf2

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Change Log
 
+# 0.5.8.2 (February 05, 2021)
+- fix: restore google pubsub topic settings
+# 0.5.8.1 (February 05, 2021)
+- fix: keep message ordering with google pubsub
+
+# 0.5.8 (January 29, 2021)
+- fix: keep message ordering with google pubsub
+
+# 0.5.7.1 (January 26, 2021)
+- fix: does not call :on_error_processing when processing a message 
+
+# 0.5.7 (January 13, 2021)
+- feat: add method to preload sync listeners
+
+# 0.5.6 (January 12, 2021)
+- feat: add payload validation
+- feat: add method to rebuild payload
+
 # 0.5.5 (January 11, 2021)
 - feat: google-pub/sub receive messages in the same order they were delivered
 

data/Gemfile

@@ -2,7 +2,7 @@ source "https://rubygems.org"
 
 gem 'rubocop', '~> 1.6.0', require: false
 gem 'bunny' # rabbit-mq
-gem 'google-cloud-pubsub' # google pub/sub
+gem 'google-cloud-pubsub', '> 2.0' # google pub/sub
 gem 'ruby-kafka' # kafka pub/sub
 
 group :test do

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.5.6)
+    pub_sub_model_sync (0.5.8.2)
       rails
 
 GEM
@@ -78,49 +78,55 @@ GEM
     diff-lcs (1.3)
     digest-crc (0.5.1)
     erubi (1.10.0)
-    faraday (0.17.3)
+    faraday (1.1.0)
       multipart-post (>= 1.2, < 3)
+      ruby2_keywords
+    gapic-common (0.3.4)
+      google-protobuf (~> 3.12, >= 3.12.2)
+      googleapis-common-protos (>= 1.3.9, < 2.0)
+      googleapis-common-protos-types (>= 1.0.4, < 2.0)
+      googleauth (~> 0.9)
+      grpc (~> 1.25)
     globalid (0.4.2)
       activesupport (>= 4.2.0)
-    google-cloud-core (1.3.2)
+    google-cloud-core (1.5.0)
       google-cloud-env (~> 1.0)
-    google-cloud-env (1.2.1)
-      faraday (~> 0.11)
-    google-cloud-pubsub (1.0.2)
+      google-cloud-errors (~> 1.0)
+    google-cloud-env (1.4.0)
+      faraday (>= 0.17.3, < 2.0)
+    google-cloud-errors (1.0.1)
+    google-cloud-pubsub (2.3.0)
       concurrent-ruby (~> 1.1)
-      google-cloud-core (~> 1.2)
-      google-gax (~> 1.7)
-      googleapis-common-protos (>= 1.3.9, < 2.0)
-      grpc-google-iam-v1 (~> 0.6.9)
-    google-gax (1.7.1)
-      google-protobuf (~> 3.2)
-      googleapis-common-protos (>= 1.3.5, < 2.0)
-      googleauth (>= 0.6.2, < 0.10.0)
-      grpc (>= 1.7.2, < 2.0)
-      rly (~> 0.2.3)
-    google-protobuf (3.11.4)
-    googleapis-common-protos (1.3.9)
-      google-protobuf (~> 3.0)
-      googleapis-common-protos-types (~> 1.0)
-      grpc (~> 1.0)
-    googleapis-common-protos-types (1.0.4)
-      google-protobuf (~> 3.0)
-    googleauth (0.9.0)
-      faraday (~> 0.12)
+      google-cloud-core (~> 1.5)
+      google-cloud-pubsub-v1 (~> 0.0)
+    google-cloud-pubsub-v1 (0.1.2)
+      gapic-common (~> 0.3)
+      google-cloud-errors (~> 1.0)
+      grpc-google-iam-v1 (>= 0.6.10, < 2.0)
+    google-protobuf (3.14.0-universal-darwin)
+    googleapis-common-protos (1.3.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos-types (>= 1.0.5, < 2.0)
+      grpc (~> 1.27)
+    googleapis-common-protos-types (1.0.5)
+      google-protobuf (~> 3.11)
+    googleauth (0.14.0)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.4, < 3.0)
       memoist (~> 0.16)
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
-      signet (~> 0.7)
-    grpc (1.27.0)
-      google-protobuf (~> 3.11)
+      signet (~> 0.14)
+    grpc (1.34.0-universal-darwin)
+      google-protobuf (~> 3.13)
       googleapis-common-protos-types (~> 1.0)
-    grpc-google-iam-v1 (0.6.9)
-      googleapis-common-protos (>= 1.3.1, < 2.0)
-      grpc (~> 1.0)
+    grpc-google-iam-v1 (0.6.10)
+      google-protobuf (~> 3.11)
+      googleapis-common-protos (>= 1.3.10, < 2.0)
+      grpc (~> 1.27)
     i18n (1.8.2)
       concurrent-ruby (~> 1.0)
-    jwt (2.2.1)
+    jwt (2.2.2)
     loofah (2.8.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
@@ -134,16 +140,16 @@ GEM
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
     minitest (5.14.0)
-    multi_json (1.14.1)
+    multi_json (1.15.0)
     multipart-post (2.1.1)
     nio4r (2.5.4)
     nokogiri (1.10.10)
       mini_portile2 (~> 2.4.0)
-    os (1.0.1)
+    os (1.1.1)
     parallel (1.20.1)
     parser (2.7.2.0)
       ast (~> 2.4.1)
-    public_suffix (4.0.3)
+    public_suffix (4.0.6)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -177,7 +183,6 @@ GEM
     rake (13.0.1)
     regexp_parser (2.0.1)
     rexml (3.2.4)
-    rly (0.2.3)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
       rspec-expectations (~> 3.9.0)
@@ -205,9 +210,10 @@ GEM
     ruby-kafka (1.0.0)
       digest-crc
     ruby-progressbar (1.10.1)
-    signet (0.11.0)
+    ruby2_keywords (0.0.2)
+    signet (0.14.0)
       addressable (~> 2.3)
-      faraday (~> 0.9)
+      faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
     sprockets (4.0.2)
@@ -235,7 +241,7 @@ DEPENDENCIES
   bundler
   bunny
   database_cleaner-active_record
-  google-cloud-pubsub
+  google-cloud-pubsub (> 2.0)
   pub_sub_model_sync!
   rake
   rspec

data/README.md

@@ -15,7 +15,7 @@ Add this line to your application's Gemfile:
 ```ruby
 gem 'pub_sub_model_sync'
 
-gem 'google-cloud-pubsub' # to use google pub/sub service
+gem 'google-cloud-pubsub', '>= 1.9' # to use google pub/sub service
 gem 'bunny' # to use rabbit-mq pub/sub service
 gem 'ruby-kafka' # to use apache kafka pub/sub service
 ```
@@ -59,10 +59,10 @@ And then execute: $ bundle install
     rake pub_sub_model_sync:start
     ```
     Note: Publishers do not need todo this    
-    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing outside ```rake pub_sub_model_sync:start```
+    Note2 (Rails 6+): Due to Zeitwerk, you need to load listeners manually when syncing without mentioned task (like rails console)
     ```ruby 
       # PubSubModelSync::Config.subscribers ==> []
-      Rails.application.try(:eager_load!)
+      PubSubModelSync::Runner.preload_listeners
       # PubSubModelSync::Config.subscribers ==> [#<PubSubModelSync::Subscriber:0x000.. @klass="Article", @action=:create..., ....]
     ``` 
 
@@ -308,10 +308,13 @@ config.debug = true
     
 ## TODO
 - Add alias attributes when subscribing (similar to publisher)
-- Add flag ```model.ps_processing``` to indicate that the current transaction is being processed by pub/sub
+- Add flag ```model.ps_process_payload``` to retrieve the payload used to process the pub/sub sync
 - Auto publish update only if payload has changed
 - On delete, payload must only be composed by ids
-- Change notifications into messages 
+- Feature to publish multiple message at a time with the ability to exclude similar messages by klass and action (use the last one)
+    PubSubModelSync::MessagePublisher.batch_publish({ same_keys: :use_last_as_first|:use_last|:use_first_as_last|:keep*, same_data: :use_last_as_first*|:use_last|:use_first_as_last|:keep })
+- Add DB table to use as a shield to skip publishing similar notifications or publish partial notifications (similar idea when processing notif)
+- add callback: on_message_received(payload)
 
 ## Q&A
 - Error "could not obtain a connection from the pool within 5.000 seconds"    

data/lib/pub_sub_model_sync/mock_google_service.rb

@@ -34,6 +34,14 @@ module PubSubModelSync
       def publish(*_args)
         true
       end
+
+      def publish_async(*_args)
+        yield(OpenStruct.new(succeeded?: true)) if block_given?
+      end
+
+      def enable_message_ordering!
+        true
+      end
     end
 
     def topic(*_args)

data/lib/pub_sub_model_sync/runner.rb

@@ -4,6 +4,7 @@ require 'active_support/core_ext/module'
 module PubSubModelSync
   class Runner
     class ShutDown < StandardError; end
+    delegate :preload_listeners, to: :class
     attr_accessor :connector
 
     def initialize
@@ -12,12 +13,17 @@ module PubSubModelSync
 
     def run
       trap_signals!
-      preload_framework!
+      preload_listeners
       start_listeners
     rescue ShutDown
       connector.stop
     end
 
+    def self.preload_listeners
+      Rails.application.try(:eager_load!) if defined?(Rails)
+      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
+    end
+
     private
 
     def start_listeners
@@ -31,10 +37,5 @@ module PubSubModelSync
       end
       %w[INT QUIT TERM].each { |signal| Signal.trap(signal, handler) }
     end
-
-    def preload_framework!
-      Rails.application.try(:eager_load!) if defined?(Rails)
-      Zeitwerk::Loader.eager_load_all if defined?(Zeitwerk::Loader)
-    end
   end
 end

data/lib/pub_sub_model_sync/service_base.rb

@@ -29,7 +29,7 @@ module PubSubModelSync
       if same_app_message?(payload)
         log("Skip message from same origin: #{[payload]}") if config.debug
       else
-        payload.process!
+        payload.process
       end
     rescue => e
       error = [payload, e.message, e.backtrace]

data/lib/pub_sub_model_sync/service_google.rb

@@ -7,7 +7,8 @@ end
 
 module PubSubModelSync
   class ServiceGoogle < ServiceBase
-    LISTEN_SETTINGS = { threads: { callback: 1 } }.freeze
+    LISTEN_SETTINGS = { threads: { callback: 1 }, message_ordering: true }.freeze
+    TOPIC_SETTINGS = {}.freeze
     SUBSCRIPTION_SETTINGS = { message_ordering: true }.freeze
     attr_accessor :service, :topic, :subscription, :subscriber
 
@@ -15,7 +16,8 @@ module PubSubModelSync
       @service = Google::Cloud::Pubsub.new(project: config.project,
                                            credentials: config.credentials)
       @topic = service.topic(config.topic_name) ||
-               service.create_topic(config.topic_name)
+               service.create_topic(config.topic_name, TOPIC_SETTINGS)
+      topic.enable_message_ordering!
     end
 
     def listen_messages
@@ -30,7 +32,9 @@ module PubSubModelSync
     end
 
     def publish(payload)
-      topic.publish(payload.to_json, { SERVICE_KEY => true }.merge(PUBLISH_SETTINGS))
+      topic.publish_async(payload.to_json, message_headers) do |res|
+        raise 'Failed to publish the message.' unless res.succeeded?
+      end
     end
 
     def stop
@@ -40,6 +44,10 @@ module PubSubModelSync
 
     private
 
+    def message_headers
+      { SERVICE_KEY => true, ordering_key: SERVICE_KEY }.merge(PUBLISH_SETTINGS)
+    end
+
     def subscribe_to_topic
       topic.subscription(config.subscription_key) ||
         topic.subscribe(config.subscription_key, SUBSCRIPTION_SETTINGS)

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.5.6'
+  VERSION = '0.5.8.2'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.5.6
+  version: 0.5.8.2
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2021-01-12 00:00:00.000000000 Z
+date: 2021-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails