pub_sub_model_sync 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4eda619ab984f99ffe46cfef36fa380aaa6766c1564081dcc4e5e53e6f7976e0
-  data.tar.gz: dd056d897c96bac38a20c115fee501cbb1383038483f74849bbb0e7217568163
+  metadata.gz: fd18a29b1e99da881b9c362dbc5541f6aeee0211988bcf301336691ac071e96d
+  data.tar.gz: cbc025ff0301e1bf4fd35d78fa2b70f06ff8a3c7c1dbfa12448f1cd3a53e592f
 SHA512:
-  metadata.gz: 75796a135c2f820d89303c29c9bb6dc78eaa1e207391a07ba86a45f547a7007b4b00f767bb949d354c5a90c504b5d2d287619fd3106bca1dce71e75588ac6173
-  data.tar.gz: fe4829f970142bfea0089beb4e86a27c9d1e08049213372ae77f94b9d02d16752218d58e88b25ac1f34848931da3651e4405caa3a8932401c1410d7f04e3d0da
+  metadata.gz: 3036baf17c9f78c6b4b445eb5a3a2f7be63c2a78cb36dcf9aad1abcf1eff8b691979cd1ff79e1bac916b44b1996e0f80bedf841be102ad1ec29684c1437910a2
+  data.tar.gz: aead45d302ed53d84457376dc9fa64f05d267b7441125722d4621d4d2fc9d7097fa24f3aa693c50862b616f5a25b32e77a202a0503143d347f80290728701680

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Change Log
 
+# 0.1.5
+- Add apache kafka support
+- Add Service interface for future references
+- Improve Services to use a single/common message performer
+
 # 0.1.4
 - Add attribute aliases when publishing, ```ps_publish(['name:full_name', 'email'])```
 - Ability to retrieve publisher/subscriber crud settings

data/Gemfile

@@ -3,6 +3,7 @@ source "https://rubygems.org"
 gem 'rubocop'
 gem 'bunny' # rabbit-mq
 gem 'google-cloud-pubsub' # google pub/sub
+gem 'ruby-kafka' # kafka pub/sub
 
 # Specify your gem's dependencies in pub_sub_model_sync.gemspec
 gemspec

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    pub_sub_model_sync (0.1.4)
+    pub_sub_model_sync (0.1.5)
       activesupport
       rails
 
@@ -60,6 +60,7 @@ GEM
     concurrent-ruby (1.1.6)
     crass (1.0.6)
     diff-lcs (1.3)
+    digest-crc (0.5.1)
     erubi (1.9.0)
     faraday (0.17.3)
       multipart-post (>= 1.2, < 3)
@@ -81,7 +82,7 @@ GEM
       googleauth (>= 0.6.2, < 0.10.0)
       grpc (>= 1.7.2, < 2.0)
       rly (~> 0.2.3)
-    google-protobuf (3.11.4-universal-darwin)
+    google-protobuf (3.11.4)
     googleapis-common-protos (1.3.9)
       google-protobuf (~> 3.0)
       googleapis-common-protos-types (~> 1.0)
@@ -95,7 +96,7 @@ GEM
       multi_json (~> 1.11)
       os (>= 0.9, < 2.0)
       signet (~> 0.7)
-    grpc (1.27.0-universal-darwin)
+    grpc (1.27.0)
       google-protobuf (~> 3.11)
       googleapis-common-protos-types (~> 1.0)
     grpc-google-iam-v1 (0.6.9)
@@ -180,6 +181,8 @@ GEM
       rexml
       ruby-progressbar (~> 1.7)
       unicode-display_width (>= 1.4.0, < 1.7)
+    ruby-kafka (1.0.0)
+      digest-crc
     ruby-progressbar (1.10.1)
     signet (0.11.0)
       addressable (~> 2.3)
@@ -214,6 +217,7 @@ DEPENDENCIES
   rake
   rspec
   rubocop
+  ruby-kafka
   sqlite3
 
 BUNDLED WITH

data/README.md

@@ -1,5 +1,5 @@
 # PubSubModelSync
-Permit to sync models data and make calls between rails apps using google or rabbitmq pub/sub service. 
+Permit to sync models data and make calls between rails apps using google or rabbitmq or apache kafka pub/sub service. 
 
 Note: This gem is based on [MultipleMan](https://github.com/influitive/multiple_man) which for now looks unmaintained.
 
@@ -14,8 +14,10 @@ Note: This gem is based on [MultipleMan](https://github.com/influitive/multiple_
 Add this line to your application's Gemfile:
 ```ruby
 gem 'pub_sub_model_sync'
+
 gem 'google-cloud-pubsub' # to use google pub/sub service
 gem 'bunny' # to use rabbit-mq pub/sub service
+gem 'ruby-kafka' # to use apache kafka pub/sub service
 ```
 And then execute: $ bundle install
 
@@ -43,6 +45,14 @@ And then execute: $ bundle install
     ```
     See details here: https://github.com/ruby-amqp/bunny
 
+- configuration for Apache Kafka (You need kafka installed)
+    ```ruby
+    PubSubModelSync::Config.service_name = :kafka
+    PubSubModelSync::Config.kafka_connection = [["kafka1:9092", "kafka2:9092"], logger: Rails.logger]
+    PubSubModelSync::Config.topic_name = 'sample-topic'
+    ```
+    See details here: https://github.com/zendesk/ruby-kafka    
+
 - Add publishers/subscribers to your models (See examples below)
 
 - Start subscribers to listen for publishers (Only in the app that has subscribers)
@@ -74,6 +84,7 @@ end
 # Samples
 User.create(name: 'test user', email: 'sample@gmail.com') # Review your App 2 to see the created user (only name will be saved)
 User.ps_class_publish({ msg: 'Hello' }, action: :greeting) # User.greeting method (Class method) will be called in App2
+PubSubModelSync::Publisher.new.publish_data(User, { msg: 'Hello' }, :greeting) # similar to above when not included publisher concern
 ```
 
 ## Advanced Example
@@ -109,15 +120,22 @@ end
       # when using google service
       require 'pub_sub_model_sync/mock_google_service'
       config.before(:each) do
-        pub_sub_mock = PubSubModelSync::MockGoogleService.new
-        allow(Google::Cloud::Pubsub).to receive(:new).and_return(pub_sub_mock)
+        google_mock = PubSubModelSync::MockGoogleService.new
+        allow(Google::Cloud::Pubsub).to receive(:new).and_return(google_mock)
       end
       
       # when using rabbitmq service
       require 'pub_sub_model_sync/mock_rabbit_service' 
       config.before(:each) do
-        bunny_mock = PubSubModelSync::MockRabbitService.new
-        allow(Bunny).to receive(:new).and_return(bunny_mock)
+        rabbit_mock = PubSubModelSync::MockRabbitService.new
+        allow(Bunny).to receive(:new).and_return(rabbit_mock)
+      end
+    
+      # when using apache kafka service
+      require 'pub_sub_model_sync/mock_kafka_service' 
+      config.before(:each) do
+        kafka_mock = PubSubModelSync::MockKafkaService.new
+        allow(Kafka).to receive(:new).and_return(kafka_mock)
       end
   
     ```

data/lib/pub_sub_model_sync.rb

@@ -11,8 +11,11 @@ require 'pub_sub_model_sync/publisher_concern'
 require 'pub_sub_model_sync/runner'
 require 'pub_sub_model_sync/connector'
 require 'pub_sub_model_sync/message_processor'
+
+require 'pub_sub_model_sync/service_base'
 require 'pub_sub_model_sync/service_google'
 require 'pub_sub_model_sync/service_rabbit'
+require 'pub_sub_model_sync/service_kafka'
 
 module PubSubModelSync
   class Error < StandardError; end

data/lib/pub_sub_model_sync/config.rb

@@ -13,6 +13,9 @@ module PubSubModelSync
     # rabbitmq service
     cattr_accessor :bunny_connection, :queue_name, :topic_name
 
+    # kafka service
+    cattr_accessor :kafka_connection, :topic_name
+
     def self.log(msg, kind = :info)
       msg = "PS_MSYNC ==> #{msg}"
       if logger == :raise_error

data/lib/pub_sub_model_sync/connector.rb

@@ -16,6 +16,8 @@ module PubSubModelSync
       case Config.service_name
       when :google
         PubSubModelSync::ServiceGoogle.new
+      when :kafka
+        PubSubModelSync::ServiceKafka.new
       else # :rabbit_mq
         PubSubModelSync::ServiceRabbit.new
       end

data/lib/pub_sub_model_sync/mock_kafka_service.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class MockKafkaService
+    class MockProducer
+      def produce(*_args)
+        true
+      end
+
+      def deliver_messages(*_args)
+        true
+      end
+
+      def shutdown
+        true
+      end
+    end
+
+    class MockConsumer
+      def each_message(*_args)
+        true
+      end
+
+      def stop(*_args)
+        true
+      end
+
+      def subscribe(*_args)
+        true
+      end
+    end
+
+    def producer(*_args)
+      MockProducer.new
+    end
+
+    def consumer(*_args)
+      MockConsumer.new
+    end
+
+    def close
+      true
+    end
+  end
+end

data/lib/pub_sub_model_sync/publisher.rb

@@ -25,8 +25,7 @@ module PubSubModelSync
       {
         klass: klass.to_s,
         action: action.to_sym,
-        id: id,
-        service_model_sync: true
+        id: id
       }
     end
 

data/lib/pub_sub_model_sync/service_base.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module PubSubModelSync
+  class ServiceBase
+    SERVICE_KEY = 'service_model_sync'
+
+    def listen_messages
+      raise 'method :listen_messages must be defined in service'
+    end
+
+    def publish(_data, _attributes)
+      raise 'method :publish must be defined in service'
+    end
+
+    def stop
+      raise 'method :stop must be defined in service'
+    end
+
+    private
+
+    # @param payload (String JSON): '{"data":{},"attributes":{..}}'
+    #   refer: PubSubModelSync::Publisher (.publish_model | .publish_data)
+    def perform_message(payload)
+      data, attrs = parse_message_payload(payload)
+      args = [data, attrs[:klass], attrs[:action], attrs]
+      PubSubModelSync::MessageProcessor.new(*args).process
+    end
+
+    def parse_message_payload(payload)
+      message_payload = JSON.parse(payload).symbolize_keys
+      data = message_payload[:data].symbolize_keys
+      attrs = message_payload[:attributes].symbolize_keys
+      [data, attrs]
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_google.rb

@@ -6,7 +6,7 @@ rescue LoadError # rubocop:disable Lint/SuppressedException
 end
 
 module PubSubModelSync
-  class ServiceGoogle
+  class ServiceGoogle < ServiceBase
     attr_accessor :service, :topic, :subscription, :config, :subscriber
 
     def initialize
@@ -30,7 +30,9 @@ module PubSubModelSync
 
     def publish(data, attributes)
       log("Publishing message: #{[data, attributes]}")
-      topic.publish(data.to_json, attributes)
+
+      payload = { data: data, attributes: attributes }.to_json
+      topic.publish(payload, { SERVICE_KEY => true })
     end
 
     def stop
@@ -47,12 +49,9 @@ module PubSubModelSync
 
     def process_message(received_message)
       message = received_message.message
-      attrs = message.attributes.symbolize_keys
-      return unless attrs[:service_model_sync]
+      return unless message.attributes[SERVICE_KEY]
 
-      data = JSON.parse(message.data).symbolize_keys
-      args = [data, attrs[:klass], attrs[:action], attrs]
-      PubSubModelSync::MessageProcessor.new(*args).process
+      perform_message(message.data)
     rescue => e
       log("Error processing message: #{[received_message, e.message]}", :error)
     ensure

data/lib/pub_sub_model_sync/service_kafka.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+begin
+  require 'kafka'
+rescue LoadError # rubocop:disable Lint/SuppressedException
+end
+
+module PubSubModelSync
+  class ServiceKafka < ServiceBase
+    cattr_accessor :producer
+
+    attr_accessor :service, :consumer
+    attr_accessor :config
+    CONSUMER_GROUP = 'service_model_sync'
+
+    def initialize
+      @config = PubSubModelSync::Config
+      @service = Kafka.new(*config.kafka_connection)
+    end
+
+    def listen_messages
+      log('Listener starting...')
+      start_consumer
+      consumer.each_message(&method(:process_message))
+    rescue PubSubModelSync::Runner::ShutDown
+      raise
+    rescue => e
+      log("Error listening message: #{[e.message, e.backtrace]}", :error)
+    end
+
+    def publish(data, attributes)
+      log("Publishing: #{[data, attributes]}")
+      payload = { data: data, attributes: attributes }
+      producer.produce(payload.to_json, message_settings)
+      producer.deliver_messages
+    rescue => e
+      info = [data, attributes, e.message, e.backtrace]
+      log("Error publishing: #{info}", :error)
+    end
+
+    def stop
+      log('Listener stopping...')
+      consumer.stop
+    end
+
+    private
+
+    def message_settings
+      { topic: config.topic_name, headers: { SERVICE_KEY => true } }
+    end
+
+    def start_consumer
+      @consumer = service.consumer(group_id: CONSUMER_GROUP)
+      consumer.subscribe(config.topic_name)
+    end
+
+    def producer
+      return self.class.producer if self.class.producer
+
+      at_exit { self.class.producer.shutdown }
+      self.class.producer = service.producer
+    end
+
+    def process_message(message)
+      return unless message.headers[SERVICE_KEY]
+
+      perform_message(message.value)
+    rescue => e
+      error = [message, e.message, e.backtrace]
+      log("Error processing message: #{error}", :error)
+    end
+
+    def log(msg, kind = :info)
+      config.log("Kafka Service ==> #{msg}", kind)
+    end
+  end
+end

data/lib/pub_sub_model_sync/service_rabbit.rb

@@ -6,10 +6,9 @@ rescue LoadError # rubocop:disable Lint/SuppressedException
 end
 
 module PubSubModelSync
-  class ServiceRabbit
+  class ServiceRabbit < ServiceBase
     attr_accessor :service, :channel, :queue, :topic
     attr_accessor :config
-    SERVICE_KEY = 'service_model_sync'
 
     def initialize
       @config = PubSubModelSync::Config
@@ -31,7 +30,7 @@ module PubSubModelSync
       log("Publishing: #{[data, attributes]}")
       subscribe_to_queue
       payload = { data: data, attributes: attributes }
-      topic.publish(payload.to_json, routing_key: queue.name, type: SERVICE_KEY)
+      topic.publish(payload.to_json, message_settings)
     rescue => e
       info = [data, attributes, e.message, e.backtrace]
       log("Error publishing: #{info}", :error)
@@ -44,24 +43,19 @@ module PubSubModelSync
 
     private
 
+    def message_settings
+      { routing_key: queue.name, type: SERVICE_KEY }
+    end
+
     def process_message(_delivery_info, meta_info, payload)
       return unless meta_info[:type] == SERVICE_KEY
 
-      data, attrs = parse_message_payload(payload)
-      args = [data, attrs[:klass], attrs[:action], attrs]
-      PubSubModelSync::MessageProcessor.new(*args).process
+      perform_message(payload)
     rescue => e
       error = [payload, e.message, e.backtrace]
       log("Error processing message: #{error}", :error)
     end
 
-    def parse_message_payload(payload)
-      message_payload = JSON.parse(payload).symbolize_keys
-      data = message_payload[:data].symbolize_keys
-      attrs = message_payload[:attributes].symbolize_keys
-      [data, attrs]
-    end
-
     def subscribe_to_queue
       service.start
       @channel = service.create_channel

data/lib/pub_sub_model_sync/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module PubSubModelSync
-  VERSION = '0.1.4'
+  VERSION = '0.1.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pub_sub_model_sync
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Owen
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-03-25 00:00:00.000000000 Z
+date: 2020-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -119,12 +119,15 @@ files:
 - lib/pub_sub_model_sync/connector.rb
 - lib/pub_sub_model_sync/message_processor.rb
 - lib/pub_sub_model_sync/mock_google_service.rb
+- lib/pub_sub_model_sync/mock_kafka_service.rb
 - lib/pub_sub_model_sync/mock_rabbit_service.rb
 - lib/pub_sub_model_sync/publisher.rb
 - lib/pub_sub_model_sync/publisher_concern.rb
 - lib/pub_sub_model_sync/railtie.rb
 - lib/pub_sub_model_sync/runner.rb
+- lib/pub_sub_model_sync/service_base.rb
 - lib/pub_sub_model_sync/service_google.rb
+- lib/pub_sub_model_sync/service_kafka.rb
 - lib/pub_sub_model_sync/service_rabbit.rb
 - lib/pub_sub_model_sync/subscriber_concern.rb
 - lib/pub_sub_model_sync/tasks/worker.rake