pub_sub_model_sync 1.4.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1740843df35081aaeb35e2be5c1c27cf1ff4d219af0bcf9ae4394495ab8bba5e
-  data.tar.gz: ed506905c6b29b554f4a1cd100ddf4c8df2bf0a99590eb26901fae8feef862ce
+  metadata.gz: '0839d268607d201c9f77bbbca404f32aa5af02621b7582256eb384b42b9d3985'
+  data.tar.gz: 0ad3d2cfdce3ce99eb5d0b95a4292496927befaca2a8e2d83b865af411be2988
 SHA512:
-  metadata.gz: bff308ce421ae321d2a9e7b53df194d3678e5f6103eb4f2048d4ed064302c042fb7768f96a3c639190b8d452a0636fded9e72394a5ffe4731b4e24ba0b2f5b53
-  data.tar.gz: e60c4c78c5fbde7c40d68adb1ed029d74bb10d1759a27682850aae6015c84f938cbccccaadc976878afa26d97621d360937cdeb5cc028b2393facb12fa0d4831
+  metadata.gz: c86be499e21572b796060fc07f5d95191971b87366bbebe3b22737d5eb906f25a6e850f4b4b6f2c08f2790541169f36401fb67b95df0c5da365fda1f76e8751f
+  data.tar.gz: 3e91e306f83eb4ba8dd5c494aadc90b36a702935d7fb716424af80efee6fe6bce9ff895ae6231aa1c32327d7f468ed2148aa69a5f37842c10e0f50e206dad82a

data/.gitignore

@@ -6,6 +6,7 @@
 /pkg/
 /spec/reports/
 /tmp/
-
+/samples/app2/db/**/*
+/samples/app1/db/**/*
 # rspec failure tracking
 .rspec_status

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (1.4.0)
+    pub_sub_model_sync (1.5.0)
       rails
 
 GEM
@@ -99,8 +99,8 @@ GEM
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
       googleauth (~> 0.15, >= 0.15.1)
       grpc (~> 1.36)
-    globalid (0.4.2)
-      activesupport (>= 4.2.0)
+    globalid (1.0.0)
+      activesupport (>= 5.0)
     google-cloud-core (1.6.0)
       google-cloud-env (~> 1.0)
       google-cloud-errors (~> 1.0)
@@ -116,6 +116,7 @@ GEM
       google-cloud-errors (~> 1.0)
       grpc-google-iam-v1 (>= 0.6.10, < 2.0)
     google-protobuf (3.17.0)
+    google-protobuf (3.17.0-x86_64-linux)
     googleapis-common-protos (1.3.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos-types (>= 1.0.6, < 2.0)
@@ -132,6 +133,9 @@ GEM
     grpc (1.37.1)
       google-protobuf (~> 3.15)
       googleapis-common-protos-types (~> 1.0)
+    grpc (1.37.1-x86_64-linux)
+      google-protobuf (~> 3.15)
+      googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.11)
       google-protobuf (~> 3.14)
       googleapis-common-protos (>= 1.3.11, < 2.0)
@@ -139,27 +143,27 @@ GEM
     i18n (1.8.10)
       concurrent-ruby (~> 1.0)
     jwt (2.2.3)
-    loofah (2.9.1)
+    loofah (2.15.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
       mini_mime (>= 0.1.1)
-    marcel (1.0.1)
+    marcel (1.0.2)
     memoist (0.16.2)
     method_source (1.0.0)
     mini_mime (1.0.3)
     minitest (5.14.4)
     multi_json (1.15.0)
     multipart-post (2.1.1)
-    nio4r (2.5.7)
-    nokogiri (1.11.3-x86_64-darwin)
+    nio4r (2.5.8)
+    nokogiri (1.12.5-x86_64-linux)
       racc (~> 1.4)
     os (1.1.1)
     parallel (1.20.1)
     parser (3.0.1.1)
       ast (~> 2.4.1)
     public_suffix (4.0.6)
-    racc (1.5.2)
+    racc (1.6.0)
     rack (2.2.3)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
@@ -181,7 +185,7 @@ GEM
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
-    rails-html-sanitizer (1.3.0)
+    rails-html-sanitizer (1.4.2)
       loofah (~> 2.3)
     railties (6.1.3.2)
       actionpack (= 6.1.3.2)
@@ -226,19 +230,19 @@ GEM
       faraday (>= 0.17.3, < 2.0)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    sprockets (4.0.2)
+    sprockets (4.0.3)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
-    sprockets-rails (3.2.2)
-      actionpack (>= 4.0)
-      activesupport (>= 4.0)
+    sprockets-rails (3.4.2)
+      actionpack (>= 5.2)
+      activesupport (>= 5.2)
       sprockets (>= 3.0.0)
     sqlite3 (1.4.2)
-    thor (1.1.0)
+    thor (1.2.1)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
     unicode-display_width (1.7.0)
-    websocket-driver (0.7.3)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     zeitwerk (2.4.2)
@@ -260,4 +264,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   2.2.29
+   2.3.9

data/README.md

@@ -167,7 +167,7 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
     ps_class_subscribe(action, settings)
   end
   ```
-- Instance subscriptions: `ps_subscribe(action, mapping, settings)`     
+- Instance subscriptions: `ps_subscribe(action, mapping, settings, &block)`     
   When model receives the corresponding notification, `action` or `to_action` method will be called on the model. Like: `model.destroy`
   - `action` (Symbol|Array<Symbol>) Only notifications with this action name will be processed by this subscription. Sample: save|create|update|destroy|<any_other_action>    
   - `mapping` (Array<String>) Data mapping from payload data into model attributes, sample: ["email", "full_name:name"] (Note: Only these attributes will be assigned/synced to the current model)    
@@ -182,12 +182,14 @@ PubSubModelSync::Payload.new({ ids: [my_user.id] }, { klass: 'User', action: :ba
       Sample: `id: :id` will search for a model like: `model_class.where(id: payload.data[:id])`       
       Sample: `id: [:id, :email:user_email]` will search for a model like: `model_class.where(id: payload.data[:id], user_email: payload.data[:email])`       
     - `if:` (Symbol|Proc|Array<Symbol>) Method(s) or block called for the confirmation before calling the callback    
-    - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback    
+    - `unless:` (Symbol|Proc|Array<Symbol>) Method or block called for the negation before calling the callback
+  - `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_subscribe(:send_welcome, %i[email]) { |_data| puts model.email }`    
 
-- Class subscriptions: `ps_class_subscribe(action, settings)`     
+- Class subscriptions: `ps_class_subscribe(action, settings, &block)`     
   When current class receives the corresponding notification, `action` or `to_action` method will be called on the Class. Like: `User.hello(data)`
   * `action` (Symbol) Notification.action name
   * `settings` (Hash) refer ps_subscribe.settings except(:id)
+  * `&block` Block to be used as the callback method (ignored if `:to_action` is present). Sample: `ps_class_subscribe(:send_welcome) { |data| puts data }`
 
 - `ps_processing_payload` a class and instance variable that saves the current payload being processed
 

data/Rakefile

@@ -4,4 +4,3 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
-# by default skip ACK when failed in three of them

data/lib/pub_sub_model_sync/config.rb

@@ -7,13 +7,13 @@ module PubSubModelSync
 
     # customizable callbacks
     cattr_accessor(:debug) { false }
-    cattr_accessor :logger # LoggerInst
+    cattr_accessor(:logger) { Rails.logger }
     cattr_accessor(:transactions_max_buffer) { 1 }
     cattr_accessor(:skip_cache) { false }
 
     cattr_accessor(:on_before_processing) { ->(_payload, _info) {} } # return :cancel to skip
     cattr_accessor(:on_success_processing) { ->(_payload, _info) {} }
-    cattr_accessor(:on_error_processing) { ->(_exception, _info) {} }
+    cattr_accessor(:on_error_processing) { ->(exception, _info) { raise(exception) } }
     cattr_accessor(:on_before_publish) { ->(_payload) {} } # return :cancel to skip
     cattr_accessor(:on_after_publish) { ->(_payload) {} }
     cattr_accessor(:on_error_publish) { ->(_exception, _info) {} }

data/lib/pub_sub_model_sync/message_processor.rb

@@ -49,9 +49,14 @@ module PubSubModelSync
 
     # @param error (StandardError)
     def notify_error(error)
-      info = [payload, error.message, error.backtrace]
+      error_msg = 'Error processing message: '
+      error_details = [payload, error.message, error.backtrace]
       res = config.on_error_processing.call(error, { payload: payload })
-      log("Error processing message: #{info}", :error) if res != :skip_log
+      log("#{error_msg} #{error_details}", :error) if res != :skip_log
+    rescue => e
+      error_details = [payload, e.message, e.backtrace]
+      log("#{error_msg} #{error_details}", :error)
+      raise(e)
     end
 
     def lost_db_connection?(error)

data/lib/pub_sub_model_sync/mock_rabbit_service.rb

@@ -24,6 +24,10 @@ module PubSubModelSync
       def publish(*_args)
         true
       end
+
+      def channel
+        MockChannel.new
+      end
     end
 
     class MockChannel
@@ -39,6 +43,10 @@ module PubSubModelSync
       def close
         true
       end
+
+      def ack(_delivery_tag)
+        true
+      end
     end
 
     def create_channel(*_args)

data/lib/pub_sub_model_sync/payload_cache_optimizer.rb

@@ -41,8 +41,9 @@ module PubSubModelSync
     end
 
     def optimize_payload # rubocop:disable Metrics/AbcSize
-      changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys
-      invalid_keys = payload.data.keys - (changed_keys + payload.cache_settings[:required])
+      changed_keys = Hash[(payload.data.to_a - previous_payload_data.to_a)].keys.map(&:to_sym)
+      required_keys = payload.cache_settings[:required].map(&:to_sym)
+      invalid_keys = payload.data.keys - (changed_keys + required_keys)
       log("Excluding non changed attributes: #{invalid_keys} from: #{payload.inspect}") if debug?
       payload.exclude_data_attrs(invalid_keys)
     end

data/lib/pub_sub_model_sync/service_base.rb

@@ -33,25 +33,29 @@ module PubSubModelSync
     # @param (String: Payload in json format)
     def process_message(payload_info)
       payload = decode_payload(payload_info)
-      return payload.process unless same_app_message?(payload)
+      return unless payload
+      return if same_app_message?(payload)
 
-      log("Skipping message from same origin: #{[payload]}") if config.debug
-    rescue => e
-      error_payload = [payload, e.message, e.backtrace]
-      log("Error while starting to process a message: #{error_payload}", :error)
+      payload.process
     end
 
-    # @return Payload
+    # @return [Payload,Nil]
     def decode_payload(payload_info)
       payload = ::PubSubModelSync::Payload.from_payload_data(JSON.parse(payload_info))
       log("Received message: #{[payload]}") if config.debug
       payload
+    rescue => e
+      error_payload = [payload_info, e.message, e.backtrace]
+      log("Error while parsing payload: #{error_payload}", :error)
+      nil
     end
 
     # @param payload (Payload)
     def same_app_message?(payload)
       key = payload.headers[:app_key]
-      key && key == config.subscription_key
+      res = key && key == config.subscription_key
+      log("Skipping message from same origin: #{[payload]}") if res && config.debug
+      res
     end
   end
 end

data/lib/pub_sub_model_sync/service_google.rb

@@ -92,7 +92,6 @@ module PubSubModelSync
     def process_message(received_message)
       message = received_message.message
       super(message.data) if message.attributes[SERVICE_KEY]
-    ensure
       received_message.acknowledge!
     end
   end

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -8,7 +8,7 @@ end
 module PubSubModelSync
   class ServiceKafka < ServiceBase
     QTY_WORKERS = 10
-    LISTEN_SETTINGS = {}.freeze
+    LISTEN_SETTINGS = { automatically_mark_as_processed: false }.freeze
     PUBLISH_SETTINGS = {}.freeze
     PRODUCER_SETTINGS = { delivery_threshold: 200, delivery_interval: 30 }.freeze
     cattr_accessor :producer
@@ -73,6 +73,7 @@ module PubSubModelSync
 
     def process_message(message)
       super(message.value) if message.headers[SERVICE_KEY]
+      consumer.mark_message_as_processed(message)
     end
 
     # Check topic existence, create if missing topic

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -8,7 +8,7 @@ end
 module PubSubModelSync
   class ServiceRabbit < ServiceBase
     QUEUE_SETTINGS = { durable: true, auto_delete: false }.freeze
-    LISTEN_SETTINGS = { manual_ack: false }.freeze
+    LISTEN_SETTINGS = { manual_ack: true }.freeze
     PUBLISH_SETTINGS = {}.freeze
 
     # @!attribute topic_names (Array): ['Topic 1', 'Topic 2']
@@ -25,7 +25,9 @@ module PubSubModelSync
 
     def listen_messages
       log('Listener starting...')
-      subscribe_to_queues { |queue| queue.subscribe(LISTEN_SETTINGS, &method(:process_message)) }
+      subscribe_to_queues do |queue|
+        queue.subscribe(LISTEN_SETTINGS) { |info, meta, payload| process_message(queue, info, meta, payload) }
+      end
       log('Listener started')
       loop { sleep 5 }
     rescue PubSubModelSync::Runner::ShutDown
@@ -62,8 +64,9 @@ module PubSubModelSync
       }.merge(PUBLISH_SETTINGS)
     end
 
-    def process_message(_delivery_info, meta_info, payload)
+    def process_message(queue, delivery_info, meta_info, payload)
       super(payload) if meta_info[:type] == SERVICE_KEY
+      queue.channel.ack(delivery_info.delivery_tag)
     end
 
     def subscribe_to_queues(&block)

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '1.4.0'
+  VERSION = '1.5.0'
 end

data/pub_sub_model_sync.gemspec

@@ -27,7 +27,7 @@ Gem::Specification.new do |spec|
   # into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0")
-                     .reject { |f| f.match(%r{^(test|spec|features)/}) }
+                     .reject { |f| f.match(%r{^(test|spec|features|samples)/}) }
   end
   spec.bindir        = 'exe'
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.0
 platform: ruby
 authors:
 - Owen
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2022-03-02 00:00:00.000000000 Z
+date: 2022-06-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -134,92 +134,6 @@ files:
 - lib/pub_sub_model_sync/transaction.rb
 - lib/pub_sub_model_sync/version.rb
 - pub_sub_model_sync.gemspec
-- samples/README.md
-- samples/app1/Dockerfile
-- samples/app1/Gemfile
-- samples/app1/Gemfile.lock
-- samples/app1/README.md
-- samples/app1/Rakefile
-- samples/app1/app/models/application_record.rb
-- samples/app1/app/models/concerns/.keep
-- samples/app1/app/models/post.rb
-- samples/app1/app/models/user.rb
-- samples/app1/bin/bundle
-- samples/app1/bin/rails
-- samples/app1/bin/rake
-- samples/app1/bin/setup
-- samples/app1/bin/spring
-- samples/app1/config.ru
-- samples/app1/config/application.rb
-- samples/app1/config/boot.rb
-- samples/app1/config/credentials.yml.enc
-- samples/app1/config/database.yml
-- samples/app1/config/environment.rb
-- samples/app1/config/environments/development.rb
-- samples/app1/config/environments/production.rb
-- samples/app1/config/environments/test.rb
-- samples/app1/config/initializers/application_controller_renderer.rb
-- samples/app1/config/initializers/backtrace_silencers.rb
-- samples/app1/config/initializers/cors.rb
-- samples/app1/config/initializers/filter_parameter_logging.rb
-- samples/app1/config/initializers/inflections.rb
-- samples/app1/config/initializers/mime_types.rb
-- samples/app1/config/initializers/pubsub.rb
-- samples/app1/config/initializers/wrap_parameters.rb
-- samples/app1/config/locales/en.yml
-- samples/app1/config/master.key
-- samples/app1/config/puma.rb
-- samples/app1/config/routes.rb
-- samples/app1/config/spring.rb
-- samples/app1/db/migrate/20210513080700_create_users.rb
-- samples/app1/db/migrate/20210513134332_create_posts.rb
-- samples/app1/db/schema.rb
-- samples/app1/db/seeds.rb
-- samples/app1/docker-compose.yml
-- samples/app1/log/.keep
-- samples/app2/Dockerfile
-- samples/app2/Gemfile
-- samples/app2/Gemfile.lock
-- samples/app2/README.md
-- samples/app2/Rakefile
-- samples/app2/app/models/application_record.rb
-- samples/app2/app/models/concerns/.keep
-- samples/app2/app/models/customer.rb
-- samples/app2/app/models/post.rb
-- samples/app2/bin/bundle
-- samples/app2/bin/rails
-- samples/app2/bin/rake
-- samples/app2/bin/setup
-- samples/app2/bin/spring
-- samples/app2/config.ru
-- samples/app2/config/application.rb
-- samples/app2/config/boot.rb
-- samples/app2/config/credentials.yml.enc
-- samples/app2/config/database.yml
-- samples/app2/config/environment.rb
-- samples/app2/config/environments/development.rb
-- samples/app2/config/environments/production.rb
-- samples/app2/config/environments/test.rb
-- samples/app2/config/initializers/application_controller_renderer.rb
-- samples/app2/config/initializers/backtrace_silencers.rb
-- samples/app2/config/initializers/cors.rb
-- samples/app2/config/initializers/filter_parameter_logging.rb
-- samples/app2/config/initializers/inflections.rb
-- samples/app2/config/initializers/mime_types.rb
-- samples/app2/config/initializers/pubsub.rb
-- samples/app2/config/initializers/wrap_parameters.rb
-- samples/app2/config/locales/en.yml
-- samples/app2/config/master.key
-- samples/app2/config/puma.rb
-- samples/app2/config/routes.rb
-- samples/app2/config/spring.rb
-- samples/app2/db/development.sqlite3
-- samples/app2/db/migrate/20210513080956_create_customers.rb
-- samples/app2/db/migrate/20210513135203_create_posts.rb
-- samples/app2/db/schema.rb
-- samples/app2/db/seeds.rb
-- samples/app2/docker-compose.yml
-- samples/app2/log/.keep
 homepage: https://github.com/owen2345/pub_sub_model_sync
 licenses:
 - MIT

data/samples/README.md

@@ -1,50 +0,0 @@
-# Sample model sync
-This is a sample to sync information between rails applications using RabbitMQ
-
-## Installation
-1. Create manually the required network to share rabbitMQ accross Rails applications (just if not exist):   
-  ```docker network create shared_app_services```
-  
-2. Start RabbitMQ server   
-  ```cd samples/app1 && docker-compose up pubsub```
-
-3. In another tab access to App2 to listen notifications (Wait for step 2)      
-  - Access to the folder
-    `cd samples/app2`
-  
-  - Build docker and start listener (Received notifications will be printed here)       
-    ```docker-compose run listener```
-  
-  - Optional: Open another tab to access application to ensure synced data
-    ```docker-compose run listener bash -c "rails c```
-    ```ruby
-      user = User.last.inspect
-      user.posts.inspect
-    ```
-
-4. In another tab access to App1 to publish notifications (Wait for step 2)  
-  - Access to the application    
-    `cd samples/app1`    
-  
-  - Build docker and enter rails console    
-    ```docker-compose run app bash -c "rails db:migrate && rails c"```
-  
-  - Create a sample user    
-    ```ruby
-      user = User.create!(name: 'User 1', posts_attributes: [{ title: 'Post 1' }, { title: 'Post 2' }])
-    ```
-    Note: Check app2 console to see notifications (3 notifications)    
-    Note2: Access app2 console to see user and its posts
-    
-  - Update previous user
-    ```ruby
-      user.update!(name: 'User 1 changed', posts_attributes: user.posts.map { |post| { id: post.id, title: "#{post.title} changed" } })
-    ```
-    Note: Check app2 console to see notifications (3 notifications)    
-    Note2: Access app2 console to see changes for user and its posts
-    
-  - Destroy previous user
-    ```ruby
-      user.destroy!
-    ```    
-  

data/samples/app1/Dockerfile

@@ -1,13 +0,0 @@
-FROM ruby:2.7.1 AS builder
-
-# Allow apt to work with https-based sources
-RUN apt-get update -yqq && apt-get install -yqq --no-install-recommends apt-transport-https
-
-RUN mkdir /app
-WORKDIR /app
-
-# backend
-COPY Gemfile.lock Gemfile /app/
-RUN gem install bundler && bundle install
-
-COPY . /app

data/samples/app1/Gemfile

@@ -1,37 +0,0 @@
-source 'https://rubygems.org'
-git_source(:github) { |repo| "https://github.com/#{repo}.git" }
-
-# Bundle edge Rails instead: gem 'rails', github: 'rails/rails', branch: 'main'
-gem 'rails', '~> 6.1.3', '>= 6.1.3.2'
-# Use sqlite3 as the database for Active Record
-gem 'sqlite3', '~> 1.4'
-# Use Puma as the app server
-gem 'puma', '~> 5.0'
-# Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder
-# gem 'jbuilder', '~> 2.7'
-# Use Active Model has_secure_password
-# gem 'bcrypt', '~> 3.1.7'
-
-# Reduces boot times through caching; required in config/boot.rb
-gem 'bootsnap', '>= 1.4.4', require: false
-
-# Use Rack CORS for handling Cross-Origin Resource Sharing (CORS), making cross-origin AJAX possible
-# gem 'rack-cors'
-
-gem 'pub_sub_model_sync', '>= 1.0'
-gem 'bunny' # to use rabbit-mq pub/sub service
-gem 'annotate'
-
-group :development, :test do
-  # Call 'byebug' anywhere in the code to stop execution and get a debugger console
-  gem 'byebug', platforms: [:mri, :mingw, :x64_mingw]
-end
-
-group :development do
-  gem 'listen', '~> 3.3'
-  # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring
-  gem 'spring'
-end
-
-# Windows does not include zoneinfo files, so bundle the tzinfo-data gem
-gem 'tzinfo-data', platforms: [:mingw, :mswin, :x64_mingw, :jruby]