pub_sub_model_sync 0.1.5 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd18a29b1e99da881b9c362dbc5541f6aeee0211988bcf301336691ac071e96d
-  data.tar.gz: cbc025ff0301e1bf4fd35d78fa2b70f06ff8a3c7c1dbfa12448f1cd3a53e592f
+  metadata.gz: 7655730b3b89716b76be52db708b7c93ec64176cc158fc84323d4e811f6dcdf1
+  data.tar.gz: 874aa3f5c5c679fd3e7ec02550426f0824481194ebd99b1a0494180df3a22cc6
 SHA512:
-  metadata.gz: 3036baf17c9f78c6b4b445eb5a3a2f7be63c2a78cb36dcf9aad1abcf1eff8b691979cd1ff79e1bac916b44b1996e0f80bedf841be102ad1ec29684c1437910a2
-  data.tar.gz: aead45d302ed53d84457376dc9fa64f05d267b7441125722d4621d4d2fc9d7097fa24f3aa693c50862b616f5a25b32e77a202a0503143d347f80290728701680
+  metadata.gz: 9e8406293a7528fa5d697c2a6a30b4cb00cee900b349ef834fad5fd7771ce864a8338ac95bc1bcbe41568d9efa1737111720607cb0f002e684ab532e835ec651
+  data.tar.gz: 0f995774ddf1b916db02fed9051c26b106e549261877c72a718e9e23c45544dff6c9757231e094201c9d466b2c4922d8e6364f8167697820db5833d78ee85c85

data/.github/workflows/ruby.yml

@@ -0,0 +1,33 @@
+name: Ruby Gem
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+
+jobs:
+  build:
+    name: Tests and Code Style
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Ruby 2.6
+        uses: actions/setup-ruby@v1
+        with:
+          ruby-version: 2.6.x
+
+      - name: Install sqlite3
+        run: sudo apt-get install libsqlite3-dev
+
+      - name: Bundle install
+        run: |
+          gem install bundler
+          bundle install --jobs 4 --retry 3
+      - name: Tests (rspec)
+        run: |
+          bundle exec rspec
+      - name: Code style (Rubocop)
+        run: bundle exec rubocop

data/CHANGELOG.md

@@ -1,6 +1,24 @@
 # Change Log
 
-# 0.1.5
+# 0.2.4 (April 28, 2020)
+- Delegate .publish to the .publisher for better understanding
+
+# 0.2.3 (April 15, 2020)
+- Improve helper names
+- feat: perform manual sync with custom settings
+- fix for "IO timeout when reading 7 bytes" error (Rabbit)
+- style: do not print processed message when failed
+- feat: retry delivery message when failed (RabbitMQ)
+
+
+# 0.2.2 (March 27, 2020)
+- fix default value for cattr_accessor in ror < 5.2
+- add callbacks when publishing a message
+
+# 0.2.1
+- Add on demand model sync method
+
+# 0.2.0
 - Add apache kafka support
 - Add Service interface for future references
 - Improve Services to use a single/common message performer

data/Gemfile.lock

@@ -1,58 +1,71 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.1.5)
+    pub_sub_model_sync (0.2.4)
       activesupport
       rails
 
 GEM
   remote: https://rubygems.org/
   specs:
-    actioncable (5.2.4.1)
-      actionpack (= 5.2.4.1)
+    actioncable (6.0.2.2)
+      actionpack (= 6.0.2.2)
       nio4r (~> 2.0)
       websocket-driver (>= 0.6.1)
-    actionmailer (5.2.4.1)
-      actionpack (= 5.2.4.1)
-      actionview (= 5.2.4.1)
-      activejob (= 5.2.4.1)
+    actionmailbox (6.0.2.2)
+      actionpack (= 6.0.2.2)
+      activejob (= 6.0.2.2)
+      activerecord (= 6.0.2.2)
+      activestorage (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
+      mail (>= 2.7.1)
+    actionmailer (6.0.2.2)
+      actionpack (= 6.0.2.2)
+      actionview (= 6.0.2.2)
+      activejob (= 6.0.2.2)
       mail (~> 2.5, >= 2.5.4)
       rails-dom-testing (~> 2.0)
-    actionpack (5.2.4.1)
-      actionview (= 5.2.4.1)
-      activesupport (= 5.2.4.1)
+    actionpack (6.0.2.2)
+      actionview (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
       rack (~> 2.0, >= 2.0.8)
       rack-test (>= 0.6.3)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.2)
-    actionview (5.2.4.1)
-      activesupport (= 5.2.4.1)
+      rails-html-sanitizer (~> 1.0, >= 1.2.0)
+    actiontext (6.0.2.2)
+      actionpack (= 6.0.2.2)
+      activerecord (= 6.0.2.2)
+      activestorage (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
+      nokogiri (>= 1.8.5)
+    actionview (6.0.2.2)
+      activesupport (= 6.0.2.2)
       builder (~> 3.1)
       erubi (~> 1.4)
       rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.0.3)
-    activejob (5.2.4.1)
-      activesupport (= 5.2.4.1)
+      rails-html-sanitizer (~> 1.1, >= 1.2.0)
+    activejob (6.0.2.2)
+      activesupport (= 6.0.2.2)
       globalid (>= 0.3.6)
-    activemodel (5.2.4.1)
-      activesupport (= 5.2.4.1)
-    activerecord (5.2.4.1)
-      activemodel (= 5.2.4.1)
-      activesupport (= 5.2.4.1)
-      arel (>= 9.0)
-    activestorage (5.2.4.1)
-      actionpack (= 5.2.4.1)
-      activerecord (= 5.2.4.1)
+    activemodel (6.0.2.2)
+      activesupport (= 6.0.2.2)
+    activerecord (6.0.2.2)
+      activemodel (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
+    activestorage (6.0.2.2)
+      actionpack (= 6.0.2.2)
+      activejob (= 6.0.2.2)
+      activerecord (= 6.0.2.2)
       marcel (~> 0.3.1)
-    activesupport (5.2.4.1)
+    activesupport (6.0.2.2)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 0.7, < 2)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
+      zeitwerk (~> 2.2)
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
     amq-protocol (2.3.0)
-    arel (9.0.0)
     ast (2.4.0)
     builder (3.2.4)
     bunny (2.14.3)
@@ -106,7 +119,7 @@ GEM
       concurrent-ruby (~> 1.0)
     jaro_winkler (1.5.4)
     jwt (2.2.1)
-    loofah (2.4.0)
+    loofah (2.5.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.5.9)
     mail (2.7.1)
@@ -114,7 +127,7 @@ GEM
     marcel (0.3.3)
       mimemagic (~> 0.3.2)
     memoist (0.16.2)
-    method_source (0.9.2)
+    method_source (1.0.0)
     mimemagic (0.3.4)
     mini_mime (1.0.2)
     mini_portile2 (2.4.0)
@@ -132,30 +145,32 @@ GEM
     rack (2.2.2)
     rack-test (1.1.0)
       rack (>= 1.0, < 3)
-    rails (5.2.4.1)
-      actioncable (= 5.2.4.1)
-      actionmailer (= 5.2.4.1)
-      actionpack (= 5.2.4.1)
-      actionview (= 5.2.4.1)
-      activejob (= 5.2.4.1)
-      activemodel (= 5.2.4.1)
-      activerecord (= 5.2.4.1)
-      activestorage (= 5.2.4.1)
-      activesupport (= 5.2.4.1)
+    rails (6.0.2.2)
+      actioncable (= 6.0.2.2)
+      actionmailbox (= 6.0.2.2)
+      actionmailer (= 6.0.2.2)
+      actionpack (= 6.0.2.2)
+      actiontext (= 6.0.2.2)
+      actionview (= 6.0.2.2)
+      activejob (= 6.0.2.2)
+      activemodel (= 6.0.2.2)
+      activerecord (= 6.0.2.2)
+      activestorage (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
       bundler (>= 1.3.0)
-      railties (= 5.2.4.1)
+      railties (= 6.0.2.2)
       sprockets-rails (>= 2.0.0)
     rails-dom-testing (2.0.3)
       activesupport (>= 4.2.0)
       nokogiri (>= 1.6)
     rails-html-sanitizer (1.3.0)
       loofah (~> 2.3)
-    railties (5.2.4.1)
-      actionpack (= 5.2.4.1)
-      activesupport (= 5.2.4.1)
+    railties (6.0.2.2)
+      actionpack (= 6.0.2.2)
+      activesupport (= 6.0.2.2)
       method_source
       rake (>= 0.8.7)
-      thor (>= 0.19.0, < 2.0)
+      thor (>= 0.20.3, < 2.0)
     rainbow (3.0.0)
     rake (13.0.1)
     rexml (3.2.4)
@@ -189,7 +204,7 @@ GEM
       faraday (~> 0.9)
       jwt (>= 1.5, < 3.0)
       multi_json (~> 1.10)
-    sprockets (3.7.2)
+    sprockets (4.0.0)
       concurrent-ruby (~> 1.0)
       rack (> 1, < 3)
     sprockets-rails (3.2.1)
@@ -199,12 +214,13 @@ GEM
     sqlite3 (1.4.2)
     thor (1.0.1)
     thread_safe (0.3.6)
-    tzinfo (1.2.6)
+    tzinfo (1.2.7)
       thread_safe (~> 0.1)
     unicode-display_width (1.6.1)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.4)
+    zeitwerk (2.3.0)
 
 PLATFORMS
   ruby
@@ -221,4 +237,4 @@ DEPENDENCIES
   sqlite3
 
 BUNDLED WITH
-   2.0.2
+   2.1.4

data/README.md

@@ -48,7 +48,7 @@ And then execute: $ bundle install
 - configuration for Apache Kafka (You need kafka installed)
     ```ruby
     PubSubModelSync::Config.service_name = :kafka
-    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "kafka2:9092"], logger: Rails.logger]
+    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "localhost:2121"], logger: Rails.logger]
     PubSubModelSync::Config.topic_name = 'sample-topic'
     ```
     See details here: https://github.com/zendesk/ruby-kafka    
@@ -83,6 +83,8 @@ end
 
 # Samples
 User.create(name: 'test user', email: 'sample@gmail.com') # Review your App 2 to see the created user (only name will be saved)
+User.new(name: 'test user').ps_perform_sync(:create) # similar to above to perform sync on demand
+
 User.ps_class_publish({ msg: 'Hello' }, action: :greeting) # User.greeting method (Class method) will be called in App2
 PubSubModelSync::Publisher.new.publish_data(User, { msg: 'Hello' }, :greeting) # similar to above when not included publisher concern
 ```
@@ -95,9 +97,13 @@ class User < ActiveRecord::Base
   include PubSubModelSync::PublisherConcern
   ps_publish(%i[name:full_name email], actions: %i[update], as_klass: 'Client', id: :client_id)
   
-  def ps_skip_for?(_action)
+  def ps_skip_callback?(_action)
     false # here logic with action to skip push message
   end
+  
+  def ps_skip_sync?(_action)
+      false # here logic with action to skip push message
+    end
 end
 
 # App 2 (Subscriber)
@@ -113,6 +119,61 @@ class User < ActiveRecord::Base
 end
 ```
 
+Note: Be careful with collision of names
+```
+class User
+    # ps_publish %i[name_data:name name:key] # key will be replaced with name_data 
+    ps_publish %i[name_data:name key_data:key] # use alias to avoid collision
+    
+    def key_data
+      name
+    end
+end
+``` 
+
+## API
+- Permit to cancel sync called after create/update/destroy (Before initializing sync service)
+  ```model.ps_skip_callback?(action)```    
+  Note: Return true to cancel sync
+  
+- Callback called before preparing data for sync (Permit to stop sync)
+  ```model.ps_skip_sync?(action)```     
+  Note: return true to cancel sync
+  
+- Callback called before sync (After preparing data)
+  ```model.ps_before_sync(action, data_to_deliver)```  
+  Note: If the method returns ```:cancel```, the sync will be stopped (message will not be published)
+
+- Callback called after sync
+  ```model.ps_after_sync(action, data_delivered)```  
+
+- Perform sync on demand (:create, :update, :destroy):   
+  The target model will receive a notification to perform the indicated action  
+  ```my_model.ps_perform_sync(action_name, custom_settings = {})```  
+  * custom_settings: override default settings defined for action_name ({ attrs: [], as_klass: nil, id: nil })
+    
+- Class level notification:     
+  ```User.ps_class_publish(data, action: action_name, as_klass: custom_klass_name)```
+  Target class ```User.action_name``` will be called when message is received    
+  * data: (required, :hash) message value to deliver    
+  * action_name: (required, :sim) same action name as defined in ps_class_subscribe(...)    
+  * as_klass: (optional, :string) same class name as defined in ps_class_subscribe(...)
+      
+- Class level notification (Same as above: on demand call)    
+  ```PubSubModelSync::Publisher.new.publish_data(Klass_name, data, action_name)```  
+  * klass_name: (required, Class) same class name as defined in ps_class_subscribe(...)
+  * data: (required, :hash) message value to deliver    
+  * action_name: (required, :sim) same action name as defined in ps_class_subscribe(...)
+  
+- Get crud subscription configured for the class   
+  ```User.ps_subscriber(action_name)```  
+  * action_name (default :create, :sym): can be :create, :update, :destroy
+- Get crud publisher configured for the class   
+  ```User.ps_publisher(action_name)```  
+  * action_name (default :create, :sym): can be :create, :update, :destroy
+- Inspect all configured listeners
+  ```PubSubModelSync::Config.listeners```
+  
 ## Testing with RSpec
 - Config: (spec/rails_helper.rb)
     ```ruby
@@ -178,15 +239,6 @@ end
       expect_any_instance_of(publisher).to receive(:publish_data).with('User', data, action)
     end
     ```
-    
-    There are two special methods to extract crud configuration settings (attrs, id, ...):
-    
-    Subscribers: ```User.ps_subscriber```
-    
-    Publishers: ```User.ps_publisher```
-    
-    Note: Inspect all configured listeners with: 
-    ``` PubSubModelSync::Config.listeners ```
 
 ## Contributing
 

data/lib/pub_sub_model_sync/config.rb

@@ -2,9 +2,9 @@
 
 module PubSubModelSync
   class Config
-    cattr_accessor :listeners, default: []
-    cattr_accessor :publishers, default: []
-    cattr_accessor :service_name, default: :google
+    cattr_accessor(:listeners) { [] }
+    cattr_accessor(:publishers) { [] }
+    cattr_accessor(:service_name) { :google }
     cattr_accessor :logger
 
     # google service

data/lib/pub_sub_model_sync/message_processor.rb

@@ -13,10 +13,13 @@ module PubSubModelSync
     end
 
     def process
+      @failed = false
       log 'processing message'
       listeners = filter_listeners
-      eval_message(listeners) if listeners.any?
-      log 'processed message'
+      return log 'Skipped: No listeners' unless listeners.any?
+
+      eval_message(listeners)
+      log 'processed message' unless @failed
     end
 
     private
@@ -36,6 +39,7 @@ module PubSubModelSync
       model_class.send(listener[:action], data)
     rescue => e
       log("Error listener (#{listener}): #{e.message}", :error)
+      @failed = true
     end
 
     # support for: create, update, destroy
@@ -49,13 +53,13 @@ module PubSubModelSync
       end
     rescue => e
       log("Error listener (#{listener}): #{e.message}", :error)
+      @failed = true
     end
 
     def find_model(listener)
       model_class = listener[:klass].constantize
       identifier = listener[:settings][:id] || :id
-      model_class.where(identifier => attrs[:id]).first ||
-        model_class.new(identifier => attrs[:id])
+      model_class.where(identifier => attrs[:id]).first_or_initialize
     end
 
     def populate_model(model, listener)

data/lib/pub_sub_model_sync/mock_rabbit_service.rb

@@ -30,6 +30,10 @@ module PubSubModelSync
       def topic(*_args)
         @topic ||= MockTopic.new
       end
+
+      def close
+        true
+      end
     end
 
     def create_channel(*_args)

data/lib/pub_sub_model_sync/publisher.rb

@@ -3,22 +3,30 @@
 module PubSubModelSync
   class Publisher
     attr_accessor :connector
+    delegate :publish, to: :connector
+
     def initialize
       @connector = PubSubModelSync::Connector.new
     end
 
     def publish_data(klass, data, action)
       attributes = self.class.build_attrs(klass, action)
-      connector.publish(data, attributes)
+      publish(data, attributes)
     end
 
     # @param settings (Hash): { attrs: [], as_klass: nil, id: nil }
     def publish_model(model, action, settings = nil)
+      return if model.ps_skip_sync?(action)
+
       settings ||= model.class.ps_publisher_info(action)
       attributes = build_model_attrs(model, action, settings)
       data = {}
       data = build_model_data(model, settings[:attrs]) if action != :destroy
-      connector.publish(data.symbolize_keys, attributes)
+      res_before = model.ps_before_sync(action, data)
+      return if res_before == :cancel
+
+      publish(data.symbolize_keys, attributes)
+      model.ps_after_sync(action, data)
     end
 
     def self.build_attrs(klass, action, id = nil)

data/lib/pub_sub_model_sync/publisher_concern.rb

@@ -6,13 +6,32 @@ module PubSubModelSync
       base.extend(ClassMethods)
     end
 
-    # Permit to skip a publish callback
-    def ps_skip_for?(_action)
+    # Before initializing sync service (callbacks: after create/update/destroy)
+    def ps_skip_callback?(_action)
       false
     end
 
+    # before preparing data to sync
+    def ps_skip_sync?(_action)
+      false
+    end
+
+    # before delivering data
+    def ps_before_sync(_action, _data); end
+
+    # after delivering data
+    def ps_after_sync(_action, _data); end
+
+    # To perform sync on demand
+    # @param custom_settings (Hash): { attrs: [], as_klass: nil, id: nil }
+    def ps_perform_sync(action = :create, custom_settings = {})
+      service = self.class.ps_publisher_service
+      model_settings = self.class.ps_publisher_info(action) || {}
+      service.publish_model(self, action, model_settings.merge(custom_settings))
+    end
+
     module ClassMethods
-      # Permit to publish crud actions (:create, :update, :destroy)
+      # Permit to configure to publish crud actions (:create, :update, :destroy)
       # @param settings (Hash): { actions: nil, as_klass: nil, id: nil }
       def ps_publish(attrs, settings = {})
         actions = settings.delete(:actions) || %i[create update destroy]
@@ -23,17 +42,19 @@ module PubSubModelSync
         end
       end
 
+      # On demand class level publisher
+      def ps_class_publish(data, action:, as_klass: nil)
+        as_klass = (as_klass || name).to_s
+        ps_publisher_service.publish_data(as_klass, data, action.to_sym)
+      end
+
+      # Publisher info for specific action
       def ps_publisher_info(action = :create)
         PubSubModelSync::Config.publishers.select do |listener|
           listener[:klass] == name && listener[:action] == action
         end.last
       end
 
-      def ps_class_publish(data, action:, as_klass: nil)
-        as_klass = (as_klass || name).to_s
-        ps_publisher_service.publish_data(as_klass, data, action.to_sym)
-      end
-
       def ps_publisher_service
         PubSubModelSync::Publisher.new
       end
@@ -42,7 +63,7 @@ module PubSubModelSync
 
       def ps_register_callback(action, info)
         after_commit(on: action) do |model|
-          unless model.ps_skip_for?(action)
+          unless model.ps_skip_callback?(action)
             service = model.class.ps_publisher_service
             service.publish_model(model, action.to_sym, info)
           end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -28,9 +28,12 @@ module PubSubModelSync
 
     def publish(data, attributes)
       log("Publishing: #{[data, attributes]}")
-      subscribe_to_queue
-      payload = { data: data, attributes: attributes }
-      topic.publish(payload.to_json, message_settings)
+      deliver_data(data, attributes)
+    # TODO: max retry
+    rescue Timeout::Error => e
+      log("Error publishing (retrying....): #{e.message}", :error)
+      initialize
+      retry
     rescue => e
       info = [data, attributes, e.message, e.backtrace]
       log("Error publishing: #{info}", :error)
@@ -72,5 +75,16 @@ module PubSubModelSync
     def log(msg, kind = :info)
       config.log("Rabbit Service ==> #{msg}", kind)
     end
+
+    def deliver_data(data, attributes)
+      subscribe_to_queue
+      payload = { data: data, attributes: attributes }
+      topic.publish(payload.to_json, message_settings)
+
+      # Ugly fix: "IO timeout when reading 7 bytes"
+      # https://stackoverflow.com/questions/39039129/rabbitmq-timeouterror-io-timeout-when-reading-7-bytes
+      channel.close
+      service.close
+    end
   end
 end

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.1.5'
+  VERSION = '0.2.4'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.2.4
 platform: ruby
 authors:
 - Owen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-26 00:00:00.000000000 Z
+date: 2020-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -101,10 +101,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
 - ".rubocop.yml"
-- ".travis.yml"
 - CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
@@ -155,8 +155,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.8
 signing_key: 
 specification_version: 4
 summary: Permit to sync models between apps through pub/sub

data/.travis.yml

@@ -1,13 +0,0 @@
----
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - 2.3.1
-before_install: gem install bundler
-
-script:
-  - bundle install
-  # disabled cause of: An error occurred while Style/PercentLiteralDelimiters cop
-  # - bundle exec rubocop
-  - bundle exec rspec