waterdrop 1.4.2 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data.tar.gz.sig +0 -0
- data/.github/workflows/ci.yml +1 -2
- data/.gitignore +2 -0
- data/.ruby-version +1 -1
- data/CHANGELOG.md +17 -5
- data/Gemfile +9 -0
- data/Gemfile.lock +42 -29
- data/{MIT-LICENCE → MIT-LICENSE} +0 -0
- data/README.md +244 -57
- data/certs/mensfeld.pem +21 -21
- data/config/errors.yml +3 -16
- data/docker-compose.yml +1 -1
- data/lib/water_drop.rb +4 -24
- data/lib/water_drop/config.rb +41 -142
- data/lib/water_drop/contracts.rb +0 -2
- data/lib/water_drop/contracts/config.rb +8 -121
- data/lib/water_drop/contracts/message.rb +42 -0
- data/lib/water_drop/errors.rb +31 -5
- data/lib/water_drop/instrumentation/monitor.rb +16 -22
- data/lib/water_drop/instrumentation/stdout_listener.rb +113 -32
- data/lib/water_drop/patches/rdkafka_producer.rb +49 -0
- data/lib/water_drop/producer.rb +143 -0
- data/lib/water_drop/producer/async.rb +51 -0
- data/lib/water_drop/producer/buffer.rb +113 -0
- data/lib/water_drop/producer/builder.rb +63 -0
- data/lib/water_drop/producer/dummy_client.rb +32 -0
- data/lib/water_drop/producer/statistics_decorator.rb +71 -0
- data/lib/water_drop/producer/status.rb +52 -0
- data/lib/water_drop/producer/sync.rb +65 -0
- data/lib/water_drop/version.rb +1 -1
- data/waterdrop.gemspec +4 -4
- metadata +44 -45
- metadata.gz.sig +0 -0
- data/lib/water_drop/async_producer.rb +0 -26
- data/lib/water_drop/base_producer.rb +0 -57
- data/lib/water_drop/config_applier.rb +0 -52
- data/lib/water_drop/contracts/message_options.rb +0 -19
- data/lib/water_drop/sync_producer.rb +0 -24
data/lib/water_drop/version.rb
CHANGED
data/waterdrop.gemspec
CHANGED
@@ -16,14 +16,14 @@ Gem::Specification.new do |spec|
|
|
16
16
|
spec.description = spec.summary
|
17
17
|
spec.license = 'MIT'
|
18
18
|
|
19
|
-
spec.add_dependency '
|
19
|
+
spec.add_dependency 'concurrent-ruby', '>= 1.1'
|
20
20
|
spec.add_dependency 'dry-configurable', '~> 0.8'
|
21
21
|
spec.add_dependency 'dry-monitor', '~> 0.3'
|
22
|
-
spec.add_dependency 'dry-validation', '~> 1.
|
23
|
-
spec.add_dependency '
|
22
|
+
spec.add_dependency 'dry-validation', '~> 1.3'
|
23
|
+
spec.add_dependency 'rdkafka', '>= 0.6.0'
|
24
24
|
spec.add_dependency 'zeitwerk', '~> 2.1'
|
25
25
|
|
26
|
-
spec.required_ruby_version = '>= 2.
|
26
|
+
spec.required_ruby_version = '>= 2.6.0'
|
27
27
|
|
28
28
|
if $PROGRAM_NAME.end_with?('gem')
|
29
29
|
spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: waterdrop
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Maciej Mensfeld
|
@@ -11,51 +11,45 @@ cert_chain:
|
|
11
11
|
- |
|
12
12
|
-----BEGIN CERTIFICATE-----
|
13
13
|
MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
14
|
+
ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
|
15
|
+
NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
|
16
|
+
CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
|
17
|
+
ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
|
18
|
+
TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
|
19
|
+
1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
|
20
|
+
vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
|
21
|
+
qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
|
22
|
+
tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
|
23
|
+
g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
|
24
|
+
LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
|
25
|
+
BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
|
26
26
|
EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
27
|
+
c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
|
28
|
+
W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
|
29
|
+
yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
|
30
|
+
8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
|
31
|
+
/rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
|
32
|
+
fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
|
33
|
+
2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
|
34
|
+
R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
|
35
|
+
pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
|
36
36
|
-----END CERTIFICATE-----
|
37
|
-
date: 2021-
|
37
|
+
date: 2021-08-13 00:00:00.000000000 Z
|
38
38
|
dependencies:
|
39
39
|
- !ruby/object:Gem::Dependency
|
40
|
-
name:
|
40
|
+
name: concurrent-ruby
|
41
41
|
requirement: !ruby/object:Gem::Requirement
|
42
42
|
requirements:
|
43
43
|
- - ">="
|
44
44
|
- !ruby/object:Gem::Version
|
45
|
-
version: '
|
46
|
-
- - "<"
|
47
|
-
- !ruby/object:Gem::Version
|
48
|
-
version: 2.x
|
45
|
+
version: '1.1'
|
49
46
|
type: :runtime
|
50
47
|
prerelease: false
|
51
48
|
version_requirements: !ruby/object:Gem::Requirement
|
52
49
|
requirements:
|
53
50
|
- - ">="
|
54
51
|
- !ruby/object:Gem::Version
|
55
|
-
version: '
|
56
|
-
- - "<"
|
57
|
-
- !ruby/object:Gem::Version
|
58
|
-
version: 2.x
|
52
|
+
version: '1.1'
|
59
53
|
- !ruby/object:Gem::Dependency
|
60
54
|
name: dry-configurable
|
61
55
|
requirement: !ruby/object:Gem::Requirement
|
@@ -90,28 +84,28 @@ dependencies:
|
|
90
84
|
requirements:
|
91
85
|
- - "~>"
|
92
86
|
- !ruby/object:Gem::Version
|
93
|
-
version: '1.
|
87
|
+
version: '1.3'
|
94
88
|
type: :runtime
|
95
89
|
prerelease: false
|
96
90
|
version_requirements: !ruby/object:Gem::Requirement
|
97
91
|
requirements:
|
98
92
|
- - "~>"
|
99
93
|
- !ruby/object:Gem::Version
|
100
|
-
version: '1.
|
94
|
+
version: '1.3'
|
101
95
|
- !ruby/object:Gem::Dependency
|
102
|
-
name:
|
96
|
+
name: rdkafka
|
103
97
|
requirement: !ruby/object:Gem::Requirement
|
104
98
|
requirements:
|
105
99
|
- - ">="
|
106
100
|
- !ruby/object:Gem::Version
|
107
|
-
version: 0.
|
101
|
+
version: 0.6.0
|
108
102
|
type: :runtime
|
109
103
|
prerelease: false
|
110
104
|
version_requirements: !ruby/object:Gem::Requirement
|
111
105
|
requirements:
|
112
106
|
- - ">="
|
113
107
|
- !ruby/object:Gem::Version
|
114
|
-
version: 0.
|
108
|
+
version: 0.6.0
|
115
109
|
- !ruby/object:Gem::Dependency
|
116
110
|
name: zeitwerk
|
117
111
|
requirement: !ruby/object:Gem::Requirement
|
@@ -144,24 +138,29 @@ files:
|
|
144
138
|
- CHANGELOG.md
|
145
139
|
- Gemfile
|
146
140
|
- Gemfile.lock
|
147
|
-
- MIT-
|
141
|
+
- MIT-LICENSE
|
148
142
|
- README.md
|
149
143
|
- certs/mensfeld.pem
|
150
144
|
- config/errors.yml
|
151
145
|
- docker-compose.yml
|
152
146
|
- lib/water_drop.rb
|
153
|
-
- lib/water_drop/async_producer.rb
|
154
|
-
- lib/water_drop/base_producer.rb
|
155
147
|
- lib/water_drop/config.rb
|
156
|
-
- lib/water_drop/config_applier.rb
|
157
148
|
- lib/water_drop/contracts.rb
|
158
149
|
- lib/water_drop/contracts/config.rb
|
159
|
-
- lib/water_drop/contracts/
|
150
|
+
- lib/water_drop/contracts/message.rb
|
160
151
|
- lib/water_drop/errors.rb
|
161
152
|
- lib/water_drop/instrumentation.rb
|
162
153
|
- lib/water_drop/instrumentation/monitor.rb
|
163
154
|
- lib/water_drop/instrumentation/stdout_listener.rb
|
164
|
-
- lib/water_drop/
|
155
|
+
- lib/water_drop/patches/rdkafka_producer.rb
|
156
|
+
- lib/water_drop/producer.rb
|
157
|
+
- lib/water_drop/producer/async.rb
|
158
|
+
- lib/water_drop/producer/buffer.rb
|
159
|
+
- lib/water_drop/producer/builder.rb
|
160
|
+
- lib/water_drop/producer/dummy_client.rb
|
161
|
+
- lib/water_drop/producer/statistics_decorator.rb
|
162
|
+
- lib/water_drop/producer/status.rb
|
163
|
+
- lib/water_drop/producer/sync.rb
|
165
164
|
- lib/water_drop/version.rb
|
166
165
|
- lib/waterdrop.rb
|
167
166
|
- log/.gitkeep
|
@@ -178,14 +177,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
178
177
|
requirements:
|
179
178
|
- - ">="
|
180
179
|
- !ruby/object:Gem::Version
|
181
|
-
version: 2.
|
180
|
+
version: 2.6.0
|
182
181
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
183
182
|
requirements:
|
184
183
|
- - ">="
|
185
184
|
- !ruby/object:Gem::Version
|
186
185
|
version: '0'
|
187
186
|
requirements: []
|
188
|
-
rubygems_version: 3.2.
|
187
|
+
rubygems_version: 3.2.25
|
189
188
|
signing_key:
|
190
189
|
specification_version: 4
|
191
190
|
summary: Kafka messaging made easy!
|
metadata.gz.sig
CHANGED
Binary file
|
@@ -1,26 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# WaterDrop library
|
4
|
-
module WaterDrop
|
5
|
-
# Async producer for messages
|
6
|
-
class AsyncProducer < BaseProducer
|
7
|
-
# Performs message delivery using deliver_async method
|
8
|
-
# @param message [String] message that we want to send to Kafka
|
9
|
-
# @param options [Hash] options (including topic) for producer
|
10
|
-
# @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
|
11
|
-
# somehow invalid and we cannot perform delivery because of that
|
12
|
-
def self.call(message, options)
|
13
|
-
attempts_count ||= 0
|
14
|
-
attempts_count += 1
|
15
|
-
|
16
|
-
validate!(options)
|
17
|
-
return unless WaterDrop.config.deliver
|
18
|
-
|
19
|
-
d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
|
20
|
-
|
21
|
-
DeliveryBoy.send(d_method, message, **options)
|
22
|
-
rescue Kafka::Error => e
|
23
|
-
graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
|
24
|
-
end
|
25
|
-
end
|
26
|
-
end
|
@@ -1,57 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Base messages producer that contains all the logic that is exactly the same for both
|
5
|
-
# sync and async producers
|
6
|
-
class BaseProducer
|
7
|
-
# Contract for checking the correctness of the provided data that someone wants to
|
8
|
-
# dispatch to Kafka
|
9
|
-
SCHEMA = Contracts::MessageOptions.new.freeze
|
10
|
-
|
11
|
-
private_constant :SCHEMA
|
12
|
-
|
13
|
-
class << self
|
14
|
-
private
|
15
|
-
|
16
|
-
# Runs the message options validations and raises an error if anything is invalid
|
17
|
-
# @param options [Hash] hash that we want to validate
|
18
|
-
# @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
|
19
|
-
# somehow invalid and we cannot perform delivery because of that
|
20
|
-
def validate!(options)
|
21
|
-
validation_result = SCHEMA.call(options)
|
22
|
-
return true if validation_result.success?
|
23
|
-
|
24
|
-
raise Errors::InvalidMessageOptions, validation_result.errors
|
25
|
-
end
|
26
|
-
|
27
|
-
# Upon failed delivery, we may try to resend a message depending on the attempt number
|
28
|
-
# or re-raise an error if we're unable to do that after given number of retries
|
29
|
-
# This method checks that and also instruments errors and retries for the delivery
|
30
|
-
# @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
|
31
|
-
# @param message [String] message that we want to send to Kafka
|
32
|
-
# @param options [Hash] options (including topic) for producer
|
33
|
-
# @param error [Kafka::Error] error that occurred
|
34
|
-
# @return [Boolean] true if this is a graceful attempt and we can retry or false it this
|
35
|
-
# was the final one and we should deal with the fact, that we cannot deliver a given
|
36
|
-
# message
|
37
|
-
def graceful_attempt?(attempts_count, message, options, error)
|
38
|
-
scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
|
39
|
-
payload = {
|
40
|
-
caller: self,
|
41
|
-
message: message,
|
42
|
-
options: options,
|
43
|
-
error: error,
|
44
|
-
attempts_count: attempts_count
|
45
|
-
}
|
46
|
-
|
47
|
-
if attempts_count > WaterDrop.config.kafka.max_retries
|
48
|
-
WaterDrop.monitor.instrument("#{scope}.error", payload)
|
49
|
-
false
|
50
|
-
else
|
51
|
-
WaterDrop.monitor.instrument("#{scope}.retry", payload)
|
52
|
-
true
|
53
|
-
end
|
54
|
-
end
|
55
|
-
end
|
56
|
-
end
|
57
|
-
end
|
@@ -1,52 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
# Engine used to propagate config application to DeliveryBoy with corner case handling
|
5
|
-
module ConfigApplier
|
6
|
-
class << self
|
7
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
8
|
-
# @param settings [Hash] hash with WaterDrop settings
|
9
|
-
def call(delivery_boy_config, settings)
|
10
|
-
# Recursive lambda for mapping config down to delivery boy
|
11
|
-
settings.each do |key, value|
|
12
|
-
call(delivery_boy_config, value) && next if value.is_a?(Hash)
|
13
|
-
|
14
|
-
# If this is a special case that needs manual setup instead of a direct reassignment
|
15
|
-
if respond_to?(key, true)
|
16
|
-
send(key, delivery_boy_config, value)
|
17
|
-
else
|
18
|
-
# If this setting is our internal one, we should not sync it with the delivery boy
|
19
|
-
next unless delivery_boy_config.respond_to?(:"#{key}=")
|
20
|
-
|
21
|
-
delivery_boy_config.public_send(:"#{key}=", value)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
|
-
private
|
27
|
-
|
28
|
-
# Extra setup for the compression codec as it behaves differently than other settings
|
29
|
-
# that are ported 1:1 from ruby-kafka
|
30
|
-
# For some crazy reason, delivery boy requires compression codec as a string, when
|
31
|
-
# ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
|
32
|
-
# that by assigning a string version that down the road will be symbolized again
|
33
|
-
# by delivery boy
|
34
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
35
|
-
# @param codec_name [Symbol] codec name as a symbol
|
36
|
-
def compression_codec(delivery_boy_config, codec_name)
|
37
|
-
# If there is no compression codec, we don't apply anything
|
38
|
-
return unless codec_name
|
39
|
-
|
40
|
-
delivery_boy_config.compression_codec = codec_name.to_s
|
41
|
-
end
|
42
|
-
|
43
|
-
# We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
|
44
|
-
# manually
|
45
|
-
# @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
|
46
|
-
# @param seed_brokers [Array<String>] kafka seed brokers
|
47
|
-
def seed_brokers(delivery_boy_config, seed_brokers)
|
48
|
-
delivery_boy_config.brokers = seed_brokers
|
49
|
-
end
|
50
|
-
end
|
51
|
-
end
|
52
|
-
end
|
@@ -1,19 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module WaterDrop
|
4
|
-
module Contracts
|
5
|
-
# Contract with validation rules for validating that all the message options that
|
6
|
-
# we provide to producer ale valid and usable
|
7
|
-
# @note Does not validate message itself as it is not our concern
|
8
|
-
class MessageOptions < Dry::Validation::Contract
|
9
|
-
params do
|
10
|
-
required(:topic).filled(:str?, format?: TOPIC_REGEXP)
|
11
|
-
optional(:key).maybe(:str?, :filled?)
|
12
|
-
optional(:partition).filled(:int?, gteq?: 0)
|
13
|
-
optional(:partition_key).maybe(:str?, :filled?)
|
14
|
-
optional(:create_time).maybe(:time?)
|
15
|
-
optional(:headers).maybe(:hash?)
|
16
|
-
end
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
@@ -1,24 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
# WaterDrop library
|
4
|
-
module WaterDrop
|
5
|
-
# Sync producer for messages
|
6
|
-
class SyncProducer < BaseProducer
|
7
|
-
# Performs message delivery using deliver_async method
|
8
|
-
# @param message [String] message that we want to send to Kafka
|
9
|
-
# @param options [Hash] options (including topic) for producer
|
10
|
-
# @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
|
11
|
-
# somehow invalid and we cannot perform delivery because of that
|
12
|
-
def self.call(message, options)
|
13
|
-
attempts_count ||= 0
|
14
|
-
attempts_count += 1
|
15
|
-
|
16
|
-
validate!(options)
|
17
|
-
return unless WaterDrop.config.deliver
|
18
|
-
|
19
|
-
DeliveryBoy.deliver(message, **options)
|
20
|
-
rescue Kafka::Error => e
|
21
|
-
graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
|
22
|
-
end
|
23
|
-
end
|
24
|
-
end
|