action_mailer_kafka 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: b7345d1145d840492978828091a1a9ebe4b3c08b
-  data.tar.gz: ffdb7a81954a33a0e03bad1f3e5757403af7743a
+SHA256:
+  metadata.gz: 0ff4e4d052f93b3cdb996e0301e646ffb52f4a5722163e1c568d7ab604de9c28
+  data.tar.gz: 61188eba23f566b3a7bfd3cc435f165118b11c9f30f97d2da4787b2e91ac779b
 SHA512:
-  metadata.gz: a4db5bd4229ff0091911ac61c188f33029c4c0f67bda9b147fd6a91267b1fc435087e63b5c2e75aa875771cbeb2e65a0ee1461625583b6c7d5f5de6a3c5b8f40
-  data.tar.gz: dfd02e92da664c07175beb643cff16cdfa572df8c5be70eb90bfbf1cb9ce620e9ce07728df08b5ed271092f40b62c0ade9b1fc78c6cf23f178142d1b026f2076
+  metadata.gz: d6a92f0a54285b69b543d124099197693a0200f849e198fbd8f383cdc4395787477d923a29e6b37079334fa8556ceaf2adb4eb70310c9401f0bebef00625472b
+  data.tar.gz: 99af2dba4d7ccc063b06d70950e514fa961f8688d6d7a6bd1a8ab9cc2f23507b81f426ae1af3f327b78ea689b467736d35de2549f6b6646fa72a6a4f4f00e00d

data/.hound.yml

@@ -0,0 +1,2 @@
+rubocop:
+  config_file: .rubocop.yml

data/CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
 Changes and additions to the library will be listed here.
 
+## 1.0.0
+- Dont initialize logger by default anymore.
+- Ensure exaclty once delivery for builtin Kakfa producer.
+- Change the interface of Kafka producer.
+- Emails are packed as msgpack before being put into kafka.
+
 ## 0.1.0
 - Open source the gem. 
 

data/Gemfile

@@ -2,3 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in action_mailer_kafka.gemspec
 gemspec
+
+gem 'codecov', :require => false, :group => :test
+gem 'rubycritic', require: false

data/README.md

@@ -1,6 +1,5 @@
 # ActionMailerKafka
-[![CircleCI](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master.svg?style=svg)](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master)
-
+[![CircleCI](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master.svg?style=svg)](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master) [![Maintainability](https://api.codeclimate.com/v1/badges/4f16e7b6eb2733c52cf4/maintainability)](https://codeclimate.com/github/luong-komorebi/action-mailer-kafka/maintainability) [![codecov](https://codecov.io/gh/luong-komorebi/action-mailer-kafka/branch/master/graph/badge.svg)](https://codecov.io/gh/luong-komorebi/action-mailer-kafka)
 
 <p align="center">
   <img src="./logo.png">
@@ -42,26 +41,29 @@ config.action_mailer.delivery_method = :action_mailer_kafka
 #### Kafa settings
 The gem accepts 2 kinds of kafka setting params:
 
-1. Your Kafka publish proc
-
+1. Your Kafka Instance
 
+This allows you to configure your own Kafka instance that should inherit from `ActionMailerKafka::BaseProducer`.
 With this option, you should config as below:
 
 ```ruby
+
+class PublisherKlass < ActionMailerKafka::BaseProducer
+# your implementation
+end
+
 config.action_mailer.action_mailer_kafka_settings = {
   kafka_mail_topic: 'YourKafkaTopic',
-  kafka_publish_proc: proc do |message_data, default_message_topic|
-                          YourKafkaClientInstance.publish(message_data, default_message_topic)
-                        end
+  kafka_publisher: PublisherKlass.new
 }
 ```
 
-and the data would go through your publish process.
+and the data would go through your kafka instance.
 
 
 2. Your kafka client info
 
-With this option, the library will generate a kafka instance for you:
+Just pass in your kafka client info and we will take care of the rest. With this option, the library will generate a kafka instance for you. By default this publisher instance is optimised for idempotency and exactly once delivery in Kafka:
 
 ```ruby
 config.action_mailer.action_mailer_kafka_settings = {

data/action_mailer_kafka.gemspec

@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'mail'
+  spec.add_dependency 'msgpack', '~> 1.3'
   spec.add_dependency 'ruby-kafka', '~> 0.7.6'
   spec.add_development_dependency 'appraisal'
   spec.add_development_dependency 'bundler'
@@ -47,6 +48,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rspec-json_expectations'
   spec.add_development_dependency 'rubocop'
   spec.add_development_dependency 'rubocop-rspec'
-  spec.add_development_dependency 'rubycritic'
   spec.add_development_dependency 'simplecov'
 end

data/gemfiles/mail_2_5.gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "codecov", require: false, group: :test
+gem "rubycritic", require: false
 gem "mail", "~> 2.5.2"
 
 gemspec path: "../"

data/gemfiles/mail_2_6.gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "codecov", require: false, group: :test
+gem "rubycritic", require: false
 gem "mail", "~> 2.6.0"
 
 gemspec path: "../"

data/gemfiles/mail_2_7.gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "codecov", require: false, group: :test
+gem "rubycritic", require: false
 gem "mail", "~> 2.7.0"
 
 gemspec path: "../"

data/gemfiles/rails_4.gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "codecov", require: false, group: :test
+gem "rubycritic", require: false
 gem "rails", "~> 4.0"
 
 gemspec path: "../"

data/gemfiles/rails_5.gemfile

@@ -2,6 +2,8 @@
 
 source "https://rubygems.org"
 
+gem "codecov", require: false, group: :test
+gem "rubycritic", require: false
 gem "rails", "~> 5.0"
 
 gemspec path: "../"

data/lib/action_mailer_kafka.rb

@@ -1,8 +1,16 @@
-require 'json'
+require 'msgpack'
+MessagePack::DefaultFactory.register_type(
+  -1,
+  Time,
+  packer: MessagePack::Time::Packer
+)
+
+require 'delegate' # https://github.com/zendesk/ruby-kafka/pull/768
 require 'kafka'
 require 'mail'
 require 'action_mailer_kafka/error'
 require 'action_mailer_kafka/railtie' if defined? Rails
+require 'action_mailer_kafka/base_producer'
 require 'action_mailer_kafka/delivery_method'
 require 'action_mailer_kafka/version'
 

data/lib/action_mailer_kafka/base_producer.rb

@@ -0,0 +1,33 @@
+module ActionMailerKafka
+  class BaseProducer
+    DELIVERY_INTERVAL = 30 # trigger a delivery half a min
+    BUFFER_SIZE = 20 # trigger a delivery when buffered 20 emails
+    MAX_RETRIES = 2
+    RETRY_BACKOFF = 5
+
+    def initialize(
+      kafka_client_info:,
+      transactional_id: Socket.gethostname,
+      logger: nil
+    )
+      @logger = logger
+      kafka_client = ::Kafka.new(kafka_client_info)
+      @kafka_async_producer = kafka_client.async_producer(
+        delivery_threshold: BUFFER_SIZE,
+        delivery_interval: DELIVERY_INTERVAL,
+        max_retries: MAX_RETRIES,
+        retry_backoff: RETRY_BACKOFF,
+        idempotent: true,
+        required_acks: :all,
+        transactional_id: transactional_id
+      )
+    end
+
+    def publish(data, message_key, topic)
+      @kafka_async_producer.produce(data, key: message_key, topic: topic)
+      @kafka_async_producer.deliver_messages
+    rescue Kafka::DeliveryFailed => e
+      @logger&.error("Fail to deliver some kafka messages: #{e}")
+    end
+  end
+end

data/lib/action_mailer_kafka/delivery_method.rb

@@ -1,25 +1,23 @@
 module ActionMailerKafka
   class DeliveryMethod
     SUPPORTED_MULTIPART_MIME_TYPES = ['multipart/alternative', 'multipart/mixed', 'multipart/related'].freeze
+
     attr_accessor :settings
-    attr_reader :mailer_topic_name, :kafka_client, :kafka_publish_proc
 
     # settings params allow you to pass in
-    # 1. Your Kafka publish proc
-    # With this option, you should config as below:
-    # config.action_mailer.action_mailer_kafka_settings = {
+    # 1. Your Kafka publisher
+    # With this option, you pass an instance of Kafka Publisher, which inherit
+    # from our ActionMailerKafka::BasesProducer or at least support the method
+    # `publish` with the same parameters. After that, your should be as below:
+    # config.action_mailer.eh_mailer_settings = {
     #   kafka_mail_topic: 'YourKafkaTopic',
-    #   kafka_publish_proc: proc do |message_data, default_message_topic|
-    #                           YourKafkaClientInstance.publish(message_data,
-    #                                                           default_message_topic)
-    #                         end
+    #   kafka_publisher: PublisherKlass.new
     # }
-    #
-    # and the data would go through your publish process
+    # and the data would go through your publisher instance
     #
     # 2. Your kafka client info
     # With this option, the library will generate a kafka instance for you:
-    # config.action_mailer.action_mailer_kafka_settings = {
+    # config.action_mailer.eh_mailer_settings = {
     #   kafka_mail_topic: 'YourKafkaTopic',
     #   kafka_client_info: {
     #     seed_brokers: ['localhost:9090'],
@@ -39,48 +37,47 @@ module ActionMailerKafka
 
     def initialize(**params)
       @settings = params
-      @service_name = params[:service_name] || ''
-      @mailer_topic_name = @settings.fetch(:kafka_mail_topic)
-      if @settings[:fallback]
+      # Optional config
+      @logger = settings[:logger]
+      @raise_on_delivery_error = settings[:raise_on_delivery_error]
+
+      # General configuration
+      @service_name = settings[:service_name] || ''
+      @mailer_topic_name = settings.fetch(:kafka_mail_topic)
+      @kafka_publisher = settings[:kafka_publisher] || ActionMailerKafka::BaseProducer.new(
+        logger: @logger, kafka_client_info: settings[:kafka_client_info]
+      )
+
+      # Fallback configuration
+      @fallback = settings[:fallback]
+      if @fallback
         @fallback_delivery_method = Mail::Configuration.instance.lookup_delivery_method(
-          @settings[:fallback].fetch(:fallback_delivery_method)
+          @fallback.fetch(:fallback_delivery_method)
         ).new(
-          @settings[:fallback].fetch(:fallback_delivery_method_settings)
+          @fallback.fetch(:fallback_delivery_method_settings)
         )
       end
-      if @settings[:kafka_publish_proc]
-        @kafka_publish_proc = @settings[:kafka_publish_proc]
-      else
-        @kafka_client = ::Kafka.new(@settings.fetch(:kafka_client_info))
-        @kafka_publish_proc = proc { |data, topic|
-          kafka_client.deliver_message(data, topic: topic)
-        }
-      end
     rescue KeyError => e
-      raise RequiredParamsError.new(params, e.message)
-    end
-
-    def logger
-      @settings[:logger] || Logger.new(STDOUT)
+      raise RequiredParamsError.new(settings, e.message)
     end
 
     def deliver!(mail)
-      mail_data = construct_mail_data mail
-      kafka_publish_proc.call(mail_data, mailer_topic_name)
+      mail_data = construct_mail_as_kafka_message(mail)
+      @kafka_publisher.publish(mail_data, construct_message_key, @mailer_topic_name)
     rescue Kafka::Error => e
       error_msg = "Fail to send email into Kafka due to: #{e.message}. Delivered using fallback method"
-      logger.error(error_msg)
-      @fallback_delivery_method.deliver!(mail) if @settings[:fallback]
-      raise KafkaOperationError, error_msg if @settings[:raise_on_delivery_error]
+      @logger&.error(error_msg)
+      @fallback_delivery_method.deliver!(mail) if @fallback
+      raise KafkaOperationError, error_msg if @raise_on_delivery_error
     rescue StandardError => e
       error_msg = "Fail to send email due to: #{e.message}"
-      logger.error(error_msg)
-      raise ParsingOperationError, error_msg if @settings[:raise_on_delivery_error]
+      @logger&.error(error_msg)
+      raise ParsingOperationError, error_msg if @raise_on_delivery_error
     end
 
     private
 
-    def construct_mail_data(mail)
+    def construct_mail_as_kafka_message(mail)
       general_data = {
         subject: mail.subject,
         from: mail.from,
@@ -93,7 +90,7 @@ module ActionMailerKafka
       general_data.merge! construct_mail_body(mail)
       general_data.merge! construct_custom_mail_header(mail)
       general_data[:attachments] = construct_attachments mail
-      general_data.to_json
+      general_data.to_msgpack
     end
 
     def construct_custom_mail_header(mail)
@@ -135,5 +132,11 @@ module ActionMailerKafka
         { body: mail.body&.decoded }
       end
     end
+
+    def construct_message_key
+      # shamelessly copy from https://www.rubydoc.info/github/mikel/mail/Mail%2FUtilities:generate_message_id
+      # because some 'mail' version doesn't have this function
+      "<#{Mail.random_tag}@#{::Socket.gethostname}.mail>"
+    end
   end
 end

data/lib/action_mailer_kafka/error.rb

@@ -9,14 +9,8 @@ module ActionMailerKafka
   end
 
   class KafkaOperationError < Error
-    def initialize(msg)
-      super(msg)
-    end
   end
 
   class ParsingOperationError < Error
-    def initialize(msg)
-      super(msg)
-    end
   end
 end

data/lib/action_mailer_kafka/version.rb

@@ -1,3 +1,3 @@
 module ActionMailerKafka
-  VERSION = '0.1.0'.freeze
+  VERSION = '1.0.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: action_mailer_kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - Luong Vo
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2019-06-21 00:00:00.000000000 Z
+date: 2019-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mail
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
 - !ruby/object:Gem::Dependency
   name: ruby-kafka
   requirement: !ruby/object:Gem::Requirement
@@ -178,20 +192,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rubycritic
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -217,6 +217,7 @@ extra_rdoc_files: []
 files:
 - ".circleci/config.yml"
 - ".gitignore"
+- ".hound.yml"
 - ".rspec"
 - ".rubocop.yml"
 - Appraisals
@@ -295,6 +296,7 @@ files:
 - gemfiles/rails_4.gemfile
 - gemfiles/rails_5.gemfile
 - lib/action_mailer_kafka.rb
+- lib/action_mailer_kafka/base_producer.rb
 - lib/action_mailer_kafka/delivery_method.rb
 - lib/action_mailer_kafka/error.rb
 - lib/action_mailer_kafka/railtie.rb
@@ -320,8 +322,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.14.4
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: Custom action mailer to send mails to Kafka message queue.