action_mailer_kafka 0.1.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.hound.yml +2 -0
- data/CHANGELOG.md +6 -0
- data/Gemfile +3 -0
- data/README.md +11 -9
- data/action_mailer_kafka.gemspec +1 -1
- data/gemfiles/mail_2_5.gemfile +2 -0
- data/gemfiles/mail_2_6.gemfile +2 -0
- data/gemfiles/mail_2_7.gemfile +2 -0
- data/gemfiles/rails_4.gemfile +2 -0
- data/gemfiles/rails_5.gemfile +2 -0
- data/lib/action_mailer_kafka.rb +9 -1
- data/lib/action_mailer_kafka/base_producer.rb +33 -0
- data/lib/action_mailer_kafka/delivery_method.rb +41 -38
- data/lib/action_mailer_kafka/error.rb +0 -6
- data/lib/action_mailer_kafka/version.rb +1 -1
- metadata +19 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: 0ff4e4d052f93b3cdb996e0301e646ffb52f4a5722163e1c568d7ab604de9c28
|
4
|
+
data.tar.gz: 61188eba23f566b3a7bfd3cc435f165118b11c9f30f97d2da4787b2e91ac779b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d6a92f0a54285b69b543d124099197693a0200f849e198fbd8f383cdc4395787477d923a29e6b37079334fa8556ceaf2adb4eb70310c9401f0bebef00625472b
|
7
|
+
data.tar.gz: 99af2dba4d7ccc063b06d70950e514fa961f8688d6d7a6bd1a8ab9cc2f23507b81f426ae1af3f327b78ea689b467736d35de2549f6b6646fa72a6a4f4f00e00d
|
data/.hound.yml
ADDED
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,12 @@
|
|
1
1
|
# Changelog
|
2
2
|
Changes and additions to the library will be listed here.
|
3
3
|
|
4
|
+
## 1.0.0
|
5
|
+
- Dont initialize logger by default anymore.
|
6
|
+
- Ensure exaclty once delivery for builtin Kakfa producer.
|
7
|
+
- Change the interface of Kafka producer.
|
8
|
+
- Emails are packed as msgpack before being put into kafka.
|
9
|
+
|
4
10
|
## 0.1.0
|
5
11
|
- Open source the gem.
|
6
12
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
# ActionMailerKafka
|
2
|
-
[](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master)
|
3
|
-
|
2
|
+
[](https://circleci.com/gh/luong-komorebi/action-mailer-kafka/tree/master) [](https://codeclimate.com/github/luong-komorebi/action-mailer-kafka/maintainability) [](https://codecov.io/gh/luong-komorebi/action-mailer-kafka)
|
4
3
|
|
5
4
|
<p align="center">
|
6
5
|
<img src="./logo.png">
|
@@ -42,26 +41,29 @@ config.action_mailer.delivery_method = :action_mailer_kafka
|
|
42
41
|
#### Kafa settings
|
43
42
|
The gem accepts 2 kinds of kafka setting params:
|
44
43
|
|
45
|
-
1. Your Kafka
|
46
|
-
|
44
|
+
1. Your Kafka Instance
|
47
45
|
|
46
|
+
This allows you to configure your own Kafka instance that should inherit from `ActionMailerKafka::BaseProducer`.
|
48
47
|
With this option, you should config as below:
|
49
48
|
|
50
49
|
```ruby
|
50
|
+
|
51
|
+
class PublisherKlass < ActionMailerKafka::BaseProducer
|
52
|
+
# your implementation
|
53
|
+
end
|
54
|
+
|
51
55
|
config.action_mailer.action_mailer_kafka_settings = {
|
52
56
|
kafka_mail_topic: 'YourKafkaTopic',
|
53
|
-
|
54
|
-
YourKafkaClientInstance.publish(message_data, default_message_topic)
|
55
|
-
end
|
57
|
+
kafka_publisher: PublisherKlass.new
|
56
58
|
}
|
57
59
|
```
|
58
60
|
|
59
|
-
and the data would go through your
|
61
|
+
and the data would go through your kafka instance.
|
60
62
|
|
61
63
|
|
62
64
|
2. Your kafka client info
|
63
65
|
|
64
|
-
With this option, the library will generate a kafka instance for you:
|
66
|
+
Just pass in your kafka client info and we will take care of the rest. With this option, the library will generate a kafka instance for you. By default this publisher instance is optimised for idempotency and exactly once delivery in Kafka:
|
65
67
|
|
66
68
|
```ruby
|
67
69
|
config.action_mailer.action_mailer_kafka_settings = {
|
data/action_mailer_kafka.gemspec
CHANGED
@@ -36,6 +36,7 @@ Gem::Specification.new do |spec|
|
|
36
36
|
spec.require_paths = ['lib']
|
37
37
|
|
38
38
|
spec.add_dependency 'mail'
|
39
|
+
spec.add_dependency 'msgpack', '~> 1.3'
|
39
40
|
spec.add_dependency 'ruby-kafka', '~> 0.7.6'
|
40
41
|
spec.add_development_dependency 'appraisal'
|
41
42
|
spec.add_development_dependency 'bundler'
|
@@ -47,6 +48,5 @@ Gem::Specification.new do |spec|
|
|
47
48
|
spec.add_development_dependency 'rspec-json_expectations'
|
48
49
|
spec.add_development_dependency 'rubocop'
|
49
50
|
spec.add_development_dependency 'rubocop-rspec'
|
50
|
-
spec.add_development_dependency 'rubycritic'
|
51
51
|
spec.add_development_dependency 'simplecov'
|
52
52
|
end
|
data/gemfiles/mail_2_5.gemfile
CHANGED
data/gemfiles/mail_2_6.gemfile
CHANGED
data/gemfiles/mail_2_7.gemfile
CHANGED
data/gemfiles/rails_4.gemfile
CHANGED
data/gemfiles/rails_5.gemfile
CHANGED
data/lib/action_mailer_kafka.rb
CHANGED
@@ -1,8 +1,16 @@
|
|
1
|
-
require '
|
1
|
+
require 'msgpack'
|
2
|
+
MessagePack::DefaultFactory.register_type(
|
3
|
+
-1,
|
4
|
+
Time,
|
5
|
+
packer: MessagePack::Time::Packer
|
6
|
+
)
|
7
|
+
|
8
|
+
require 'delegate' # https://github.com/zendesk/ruby-kafka/pull/768
|
2
9
|
require 'kafka'
|
3
10
|
require 'mail'
|
4
11
|
require 'action_mailer_kafka/error'
|
5
12
|
require 'action_mailer_kafka/railtie' if defined? Rails
|
13
|
+
require 'action_mailer_kafka/base_producer'
|
6
14
|
require 'action_mailer_kafka/delivery_method'
|
7
15
|
require 'action_mailer_kafka/version'
|
8
16
|
|
@@ -0,0 +1,33 @@
|
|
1
|
+
module ActionMailerKafka
|
2
|
+
class BaseProducer
|
3
|
+
DELIVERY_INTERVAL = 30 # trigger a delivery half a min
|
4
|
+
BUFFER_SIZE = 20 # trigger a delivery when buffered 20 emails
|
5
|
+
MAX_RETRIES = 2
|
6
|
+
RETRY_BACKOFF = 5
|
7
|
+
|
8
|
+
def initialize(
|
9
|
+
kafka_client_info:,
|
10
|
+
transactional_id: Socket.gethostname,
|
11
|
+
logger: nil
|
12
|
+
)
|
13
|
+
@logger = logger
|
14
|
+
kafka_client = ::Kafka.new(kafka_client_info)
|
15
|
+
@kafka_async_producer = kafka_client.async_producer(
|
16
|
+
delivery_threshold: BUFFER_SIZE,
|
17
|
+
delivery_interval: DELIVERY_INTERVAL,
|
18
|
+
max_retries: MAX_RETRIES,
|
19
|
+
retry_backoff: RETRY_BACKOFF,
|
20
|
+
idempotent: true,
|
21
|
+
required_acks: :all,
|
22
|
+
transactional_id: transactional_id
|
23
|
+
)
|
24
|
+
end
|
25
|
+
|
26
|
+
def publish(data, message_key, topic)
|
27
|
+
@kafka_async_producer.produce(data, key: message_key, topic: topic)
|
28
|
+
@kafka_async_producer.deliver_messages
|
29
|
+
rescue Kafka::DeliveryFailed => e
|
30
|
+
@logger&.error("Fail to deliver some kafka messages: #{e}")
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -1,25 +1,23 @@
|
|
1
1
|
module ActionMailerKafka
|
2
2
|
class DeliveryMethod
|
3
3
|
SUPPORTED_MULTIPART_MIME_TYPES = ['multipart/alternative', 'multipart/mixed', 'multipart/related'].freeze
|
4
|
+
|
4
5
|
attr_accessor :settings
|
5
|
-
attr_reader :mailer_topic_name, :kafka_client, :kafka_publish_proc
|
6
6
|
|
7
7
|
# settings params allow you to pass in
|
8
|
-
# 1. Your Kafka
|
9
|
-
# With this option, you
|
10
|
-
#
|
8
|
+
# 1. Your Kafka publisher
|
9
|
+
# With this option, you pass an instance of Kafka Publisher, which inherit
|
10
|
+
# from our ActionMailerKafka::BasesProducer or at least support the method
|
11
|
+
# `publish` with the same parameters. After that, your should be as below:
|
12
|
+
# config.action_mailer.eh_mailer_settings = {
|
11
13
|
# kafka_mail_topic: 'YourKafkaTopic',
|
12
|
-
#
|
13
|
-
# YourKafkaClientInstance.publish(message_data,
|
14
|
-
# default_message_topic)
|
15
|
-
# end
|
14
|
+
# kafka_publisher: PublisherKlass.new
|
16
15
|
# }
|
17
|
-
#
|
18
|
-
# and the data would go through your publish process
|
16
|
+
# and the data would go through your publisher instance
|
19
17
|
#
|
20
18
|
# 2. Your kafka client info
|
21
19
|
# With this option, the library will generate a kafka instance for you:
|
22
|
-
# config.action_mailer.
|
20
|
+
# config.action_mailer.eh_mailer_settings = {
|
23
21
|
# kafka_mail_topic: 'YourKafkaTopic',
|
24
22
|
# kafka_client_info: {
|
25
23
|
# seed_brokers: ['localhost:9090'],
|
@@ -39,48 +37,47 @@ module ActionMailerKafka
|
|
39
37
|
|
40
38
|
def initialize(**params)
|
41
39
|
@settings = params
|
42
|
-
|
43
|
-
@
|
44
|
-
|
40
|
+
# Optional config
|
41
|
+
@logger = settings[:logger]
|
42
|
+
@raise_on_delivery_error = settings[:raise_on_delivery_error]
|
43
|
+
|
44
|
+
# General configuration
|
45
|
+
@service_name = settings[:service_name] || ''
|
46
|
+
@mailer_topic_name = settings.fetch(:kafka_mail_topic)
|
47
|
+
@kafka_publisher = settings[:kafka_publisher] || ActionMailerKafka::BaseProducer.new(
|
48
|
+
logger: @logger, kafka_client_info: settings[:kafka_client_info]
|
49
|
+
)
|
50
|
+
|
51
|
+
# Fallback configuration
|
52
|
+
@fallback = settings[:fallback]
|
53
|
+
if @fallback
|
45
54
|
@fallback_delivery_method = Mail::Configuration.instance.lookup_delivery_method(
|
46
|
-
@
|
55
|
+
@fallback.fetch(:fallback_delivery_method)
|
47
56
|
).new(
|
48
|
-
@
|
57
|
+
@fallback.fetch(:fallback_delivery_method_settings)
|
49
58
|
)
|
50
59
|
end
|
51
|
-
if @settings[:kafka_publish_proc]
|
52
|
-
@kafka_publish_proc = @settings[:kafka_publish_proc]
|
53
|
-
else
|
54
|
-
@kafka_client = ::Kafka.new(@settings.fetch(:kafka_client_info))
|
55
|
-
@kafka_publish_proc = proc { |data, topic|
|
56
|
-
kafka_client.deliver_message(data, topic: topic)
|
57
|
-
}
|
58
|
-
end
|
59
60
|
rescue KeyError => e
|
60
|
-
raise RequiredParamsError.new(
|
61
|
-
end
|
62
|
-
|
63
|
-
def logger
|
64
|
-
@settings[:logger] || Logger.new(STDOUT)
|
61
|
+
raise RequiredParamsError.new(settings, e.message)
|
65
62
|
end
|
66
63
|
|
67
64
|
def deliver!(mail)
|
68
|
-
mail_data =
|
69
|
-
|
65
|
+
mail_data = construct_mail_as_kafka_message(mail)
|
66
|
+
@kafka_publisher.publish(mail_data, construct_message_key, @mailer_topic_name)
|
70
67
|
rescue Kafka::Error => e
|
71
68
|
error_msg = "Fail to send email into Kafka due to: #{e.message}. Delivered using fallback method"
|
72
|
-
logger
|
73
|
-
@fallback_delivery_method.deliver!(mail) if @
|
74
|
-
raise KafkaOperationError, error_msg if @
|
69
|
+
@logger&.error(error_msg)
|
70
|
+
@fallback_delivery_method.deliver!(mail) if @fallback
|
71
|
+
raise KafkaOperationError, error_msg if @raise_on_delivery_error
|
75
72
|
rescue StandardError => e
|
76
73
|
error_msg = "Fail to send email due to: #{e.message}"
|
77
|
-
logger
|
78
|
-
raise ParsingOperationError, error_msg if @
|
74
|
+
@logger&.error(error_msg)
|
75
|
+
raise ParsingOperationError, error_msg if @raise_on_delivery_error
|
79
76
|
end
|
80
77
|
|
81
78
|
private
|
82
79
|
|
83
|
-
def
|
80
|
+
def construct_mail_as_kafka_message(mail)
|
84
81
|
general_data = {
|
85
82
|
subject: mail.subject,
|
86
83
|
from: mail.from,
|
@@ -93,7 +90,7 @@ module ActionMailerKafka
|
|
93
90
|
general_data.merge! construct_mail_body(mail)
|
94
91
|
general_data.merge! construct_custom_mail_header(mail)
|
95
92
|
general_data[:attachments] = construct_attachments mail
|
96
|
-
general_data.
|
93
|
+
general_data.to_msgpack
|
97
94
|
end
|
98
95
|
|
99
96
|
def construct_custom_mail_header(mail)
|
@@ -135,5 +132,11 @@ module ActionMailerKafka
|
|
135
132
|
{ body: mail.body&.decoded }
|
136
133
|
end
|
137
134
|
end
|
135
|
+
|
136
|
+
def construct_message_key
|
137
|
+
# shamelessly copy from https://www.rubydoc.info/github/mikel/mail/Mail%2FUtilities:generate_message_id
|
138
|
+
# because some 'mail' version doesn't have this function
|
139
|
+
"<#{Mail.random_tag}@#{::Socket.gethostname}.mail>"
|
140
|
+
end
|
138
141
|
end
|
139
142
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: action_mailer_kafka
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 1.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Luong Vo
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2019-
|
11
|
+
date: 2019-10-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: mail
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: msgpack
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '1.3'
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '1.3'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: ruby-kafka
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -178,20 +192,6 @@ dependencies:
|
|
178
192
|
- - ">="
|
179
193
|
- !ruby/object:Gem::Version
|
180
194
|
version: '0'
|
181
|
-
- !ruby/object:Gem::Dependency
|
182
|
-
name: rubycritic
|
183
|
-
requirement: !ruby/object:Gem::Requirement
|
184
|
-
requirements:
|
185
|
-
- - ">="
|
186
|
-
- !ruby/object:Gem::Version
|
187
|
-
version: '0'
|
188
|
-
type: :development
|
189
|
-
prerelease: false
|
190
|
-
version_requirements: !ruby/object:Gem::Requirement
|
191
|
-
requirements:
|
192
|
-
- - ">="
|
193
|
-
- !ruby/object:Gem::Version
|
194
|
-
version: '0'
|
195
195
|
- !ruby/object:Gem::Dependency
|
196
196
|
name: simplecov
|
197
197
|
requirement: !ruby/object:Gem::Requirement
|
@@ -217,6 +217,7 @@ extra_rdoc_files: []
|
|
217
217
|
files:
|
218
218
|
- ".circleci/config.yml"
|
219
219
|
- ".gitignore"
|
220
|
+
- ".hound.yml"
|
220
221
|
- ".rspec"
|
221
222
|
- ".rubocop.yml"
|
222
223
|
- Appraisals
|
@@ -295,6 +296,7 @@ files:
|
|
295
296
|
- gemfiles/rails_4.gemfile
|
296
297
|
- gemfiles/rails_5.gemfile
|
297
298
|
- lib/action_mailer_kafka.rb
|
299
|
+
- lib/action_mailer_kafka/base_producer.rb
|
298
300
|
- lib/action_mailer_kafka/delivery_method.rb
|
299
301
|
- lib/action_mailer_kafka/error.rb
|
300
302
|
- lib/action_mailer_kafka/railtie.rb
|
@@ -320,8 +322,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
320
322
|
- !ruby/object:Gem::Version
|
321
323
|
version: '0'
|
322
324
|
requirements: []
|
323
|
-
|
324
|
-
rubygems_version: 2.6.14.4
|
325
|
+
rubygems_version: 3.0.6
|
325
326
|
signing_key:
|
326
327
|
specification_version: 4
|
327
328
|
summary: Custom action mailer to send mails to Kafka message queue.
|