waterdrop 1.4.2 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.github/workflows/ci.yml +1 -2
  5. data/.gitignore +2 -0
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +17 -5
  8. data/Gemfile +9 -0
  9. data/Gemfile.lock +42 -29
  10. data/{MIT-LICENCE → MIT-LICENSE} +0 -0
  11. data/README.md +244 -57
  12. data/certs/mensfeld.pem +21 -21
  13. data/config/errors.yml +3 -16
  14. data/docker-compose.yml +1 -1
  15. data/lib/water_drop.rb +4 -24
  16. data/lib/water_drop/config.rb +41 -142
  17. data/lib/water_drop/contracts.rb +0 -2
  18. data/lib/water_drop/contracts/config.rb +8 -121
  19. data/lib/water_drop/contracts/message.rb +42 -0
  20. data/lib/water_drop/errors.rb +31 -5
  21. data/lib/water_drop/instrumentation/monitor.rb +16 -22
  22. data/lib/water_drop/instrumentation/stdout_listener.rb +113 -32
  23. data/lib/water_drop/patches/rdkafka_producer.rb +49 -0
  24. data/lib/water_drop/producer.rb +143 -0
  25. data/lib/water_drop/producer/async.rb +51 -0
  26. data/lib/water_drop/producer/buffer.rb +113 -0
  27. data/lib/water_drop/producer/builder.rb +63 -0
  28. data/lib/water_drop/producer/dummy_client.rb +32 -0
  29. data/lib/water_drop/producer/statistics_decorator.rb +71 -0
  30. data/lib/water_drop/producer/status.rb +52 -0
  31. data/lib/water_drop/producer/sync.rb +65 -0
  32. data/lib/water_drop/version.rb +1 -1
  33. data/waterdrop.gemspec +4 -4
  34. metadata +44 -45
  35. metadata.gz.sig +0 -0
  36. data/lib/water_drop/async_producer.rb +0 -26
  37. data/lib/water_drop/base_producer.rb +0 -57
  38. data/lib/water_drop/config_applier.rb +0 -52
  39. data/lib/water_drop/contracts/message_options.rb +0 -19
  40. data/lib/water_drop/sync_producer.rb +0 -24
@@ -3,5 +3,5 @@
3
3
  # WaterDrop library
4
4
  module WaterDrop
5
5
  # Current WaterDrop version
6
- VERSION = '1.4.2'
6
+ VERSION = '2.0.2'
7
7
  end
data/waterdrop.gemspec CHANGED
@@ -16,14 +16,14 @@ Gem::Specification.new do |spec|
16
16
  spec.description = spec.summary
17
17
  spec.license = 'MIT'
18
18
 
19
- spec.add_dependency 'delivery_boy', '>= 0.2', '< 2.x'
19
+ spec.add_dependency 'concurrent-ruby', '>= 1.1'
20
20
  spec.add_dependency 'dry-configurable', '~> 0.8'
21
21
  spec.add_dependency 'dry-monitor', '~> 0.3'
22
- spec.add_dependency 'dry-validation', '~> 1.2'
23
- spec.add_dependency 'ruby-kafka', '>= 0.7.8'
22
+ spec.add_dependency 'dry-validation', '~> 1.3'
23
+ spec.add_dependency 'rdkafka', '>= 0.6.0'
24
24
  spec.add_dependency 'zeitwerk', '~> 2.1'
25
25
 
26
- spec.required_ruby_version = '>= 2.5.0'
26
+ spec.required_ruby_version = '>= 2.6.0'
27
27
 
28
28
  if $PROGRAM_NAME.end_with?('gem')
29
29
  spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: waterdrop
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.4.2
4
+ version: 2.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Maciej Mensfeld
@@ -11,51 +11,45 @@ cert_chain:
11
11
  - |
12
12
  -----BEGIN CERTIFICATE-----
13
13
  MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
14
- ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
15
- NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
- CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
17
- 2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
18
- N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
19
- hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
20
- sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
21
- VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
22
- tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
23
- Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
24
- LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
- BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
14
+ ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjEwODExMTQxNTEzWhcNMjIwODExMTQx
15
+ NTEzWjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
16
+ CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDV2jKH4Ti87GM6nyT6D+ESzTI0MZDj
17
+ ak2/TEwnxvijMJyCCPKT/qIkbW4/f0VHM4rhPr1nW73sb5SZBVFCLlJcOSKOBdUY
18
+ TMY+SIXN2EtUaZuhAOe8LxtxjHTgRHvHcqUQMBENXTISNzCo32LnUxweu66ia4Pd
19
+ 1mNRhzOqNv9YiBZvtBf7IMQ+sYdOCjboq2dlsWmJiwiDpY9lQBTnWORnT3mQxU5x
20
+ vPSwnLB854cHdCS8fQo4DjeJBRZHhEbcE5sqhEMB3RZA3EtFVEXOxlNxVTS3tncI
21
+ qyNXiWDaxcipaens4ObSY1C2HTV7OWb7OMqSCIybeYTSfkaSdqmcl4S6zxXkjH1J
22
+ tnjayAVzD+QVXGijsPLE2PFnJAh9iDET2cMsjabO1f6l1OQNyAtqpcyQcgfnyW0z
23
+ g7tGxTYD+6wJHffM9d9txOUw6djkF6bDxyqB8lo4Z3IObCx18AZjI9XPS9QG7w6q
24
+ LCWuMG2lkCcRgASqaVk9fEf9yMc2xxz5o3kCAwEAAaN3MHUwCQYDVR0TBAIwADAL
25
+ BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFBqUFCKCOe5IuueUVqOB991jyCLLMB0GA1Ud
26
26
  EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
27
- c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
28
- gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
29
- 2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
30
- fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
31
- zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
32
- EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
33
- 65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
34
- 2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
35
- nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
27
+ c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBADD0/UuTTFgW+CGk2U0RDw2RBOca
28
+ W2LTF/G7AOzuzD0Tc4voc7WXyrgKwJREv8rgBimLnNlgmFJLmtUCh2U/MgxvcilH
29
+ yshYcbseNvjkrtYnLRlWZR4SSB6Zei5AlyGVQLPkvdsBpNegcG6w075YEwzX/38a
30
+ 8V9B/Yri2OGELBz8ykl7BsXUgNoUPA/4pHF6YRLz+VirOaUIQ4JfY7xGj6fSOWWz
31
+ /rQ/d77r6o1mfJYM/3BRVg73a3b7DmRnE5qjwmSaSQ7u802pJnLesmArch0xGCT/
32
+ fMmRli1Qb+6qOTl9mzD6UDMAyFR4t6MStLm0mIEqM0nBO5nUdUWbC7l9qXEf8XBE
33
+ 2DP28p3EqSuS+lKbAWKcqv7t0iRhhmaod+Yn9mcrLN1sa3q3KSQ9BCyxezCD4Mk2
34
+ R2P11bWoCtr70BsccVrN8jEhzwXngMyI2gVt750Y+dbTu1KgRqZKp/ECe7ZzPzXj
35
+ pIy9vHxTANKYVyI4qj8OrFdEM5BQNu8oQpL0iQ==
36
36
  -----END CERTIFICATE-----
37
- date: 2021-03-30 00:00:00.000000000 Z
37
+ date: 2021-08-13 00:00:00.000000000 Z
38
38
  dependencies:
39
39
  - !ruby/object:Gem::Dependency
40
- name: delivery_boy
40
+ name: concurrent-ruby
41
41
  requirement: !ruby/object:Gem::Requirement
42
42
  requirements:
43
43
  - - ">="
44
44
  - !ruby/object:Gem::Version
45
- version: '0.2'
46
- - - "<"
47
- - !ruby/object:Gem::Version
48
- version: 2.x
45
+ version: '1.1'
49
46
  type: :runtime
50
47
  prerelease: false
51
48
  version_requirements: !ruby/object:Gem::Requirement
52
49
  requirements:
53
50
  - - ">="
54
51
  - !ruby/object:Gem::Version
55
- version: '0.2'
56
- - - "<"
57
- - !ruby/object:Gem::Version
58
- version: 2.x
52
+ version: '1.1'
59
53
  - !ruby/object:Gem::Dependency
60
54
  name: dry-configurable
61
55
  requirement: !ruby/object:Gem::Requirement
@@ -90,28 +84,28 @@ dependencies:
90
84
  requirements:
91
85
  - - "~>"
92
86
  - !ruby/object:Gem::Version
93
- version: '1.2'
87
+ version: '1.3'
94
88
  type: :runtime
95
89
  prerelease: false
96
90
  version_requirements: !ruby/object:Gem::Requirement
97
91
  requirements:
98
92
  - - "~>"
99
93
  - !ruby/object:Gem::Version
100
- version: '1.2'
94
+ version: '1.3'
101
95
  - !ruby/object:Gem::Dependency
102
- name: ruby-kafka
96
+ name: rdkafka
103
97
  requirement: !ruby/object:Gem::Requirement
104
98
  requirements:
105
99
  - - ">="
106
100
  - !ruby/object:Gem::Version
107
- version: 0.7.8
101
+ version: 0.6.0
108
102
  type: :runtime
109
103
  prerelease: false
110
104
  version_requirements: !ruby/object:Gem::Requirement
111
105
  requirements:
112
106
  - - ">="
113
107
  - !ruby/object:Gem::Version
114
- version: 0.7.8
108
+ version: 0.6.0
115
109
  - !ruby/object:Gem::Dependency
116
110
  name: zeitwerk
117
111
  requirement: !ruby/object:Gem::Requirement
@@ -144,24 +138,29 @@ files:
144
138
  - CHANGELOG.md
145
139
  - Gemfile
146
140
  - Gemfile.lock
147
- - MIT-LICENCE
141
+ - MIT-LICENSE
148
142
  - README.md
149
143
  - certs/mensfeld.pem
150
144
  - config/errors.yml
151
145
  - docker-compose.yml
152
146
  - lib/water_drop.rb
153
- - lib/water_drop/async_producer.rb
154
- - lib/water_drop/base_producer.rb
155
147
  - lib/water_drop/config.rb
156
- - lib/water_drop/config_applier.rb
157
148
  - lib/water_drop/contracts.rb
158
149
  - lib/water_drop/contracts/config.rb
159
- - lib/water_drop/contracts/message_options.rb
150
+ - lib/water_drop/contracts/message.rb
160
151
  - lib/water_drop/errors.rb
161
152
  - lib/water_drop/instrumentation.rb
162
153
  - lib/water_drop/instrumentation/monitor.rb
163
154
  - lib/water_drop/instrumentation/stdout_listener.rb
164
- - lib/water_drop/sync_producer.rb
155
+ - lib/water_drop/patches/rdkafka_producer.rb
156
+ - lib/water_drop/producer.rb
157
+ - lib/water_drop/producer/async.rb
158
+ - lib/water_drop/producer/buffer.rb
159
+ - lib/water_drop/producer/builder.rb
160
+ - lib/water_drop/producer/dummy_client.rb
161
+ - lib/water_drop/producer/statistics_decorator.rb
162
+ - lib/water_drop/producer/status.rb
163
+ - lib/water_drop/producer/sync.rb
165
164
  - lib/water_drop/version.rb
166
165
  - lib/waterdrop.rb
167
166
  - log/.gitkeep
@@ -178,14 +177,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
178
177
  requirements:
179
178
  - - ">="
180
179
  - !ruby/object:Gem::Version
181
- version: 2.5.0
180
+ version: 2.6.0
182
181
  required_rubygems_version: !ruby/object:Gem::Requirement
183
182
  requirements:
184
183
  - - ">="
185
184
  - !ruby/object:Gem::Version
186
185
  version: '0'
187
186
  requirements: []
188
- rubygems_version: 3.2.3
187
+ rubygems_version: 3.2.25
189
188
  signing_key:
190
189
  specification_version: 4
191
190
  summary: Kafka messaging made easy!
metadata.gz.sig CHANGED
Binary file
@@ -1,26 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # WaterDrop library
4
- module WaterDrop
5
- # Async producer for messages
6
- class AsyncProducer < BaseProducer
7
- # Performs message delivery using deliver_async method
8
- # @param message [String] message that we want to send to Kafka
9
- # @param options [Hash] options (including topic) for producer
10
- # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
11
- # somehow invalid and we cannot perform delivery because of that
12
- def self.call(message, options)
13
- attempts_count ||= 0
14
- attempts_count += 1
15
-
16
- validate!(options)
17
- return unless WaterDrop.config.deliver
18
-
19
- d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
20
-
21
- DeliveryBoy.send(d_method, message, **options)
22
- rescue Kafka::Error => e
23
- graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
24
- end
25
- end
26
- end
@@ -1,57 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Base messages producer that contains all the logic that is exactly the same for both
5
- # sync and async producers
6
- class BaseProducer
7
- # Contract for checking the correctness of the provided data that someone wants to
8
- # dispatch to Kafka
9
- SCHEMA = Contracts::MessageOptions.new.freeze
10
-
11
- private_constant :SCHEMA
12
-
13
- class << self
14
- private
15
-
16
- # Runs the message options validations and raises an error if anything is invalid
17
- # @param options [Hash] hash that we want to validate
18
- # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
19
- # somehow invalid and we cannot perform delivery because of that
20
- def validate!(options)
21
- validation_result = SCHEMA.call(options)
22
- return true if validation_result.success?
23
-
24
- raise Errors::InvalidMessageOptions, validation_result.errors
25
- end
26
-
27
- # Upon failed delivery, we may try to resend a message depending on the attempt number
28
- # or re-raise an error if we're unable to do that after given number of retries
29
- # This method checks that and also instruments errors and retries for the delivery
30
- # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
31
- # @param message [String] message that we want to send to Kafka
32
- # @param options [Hash] options (including topic) for producer
33
- # @param error [Kafka::Error] error that occurred
34
- # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
35
- # was the final one and we should deal with the fact, that we cannot deliver a given
36
- # message
37
- def graceful_attempt?(attempts_count, message, options, error)
38
- scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
39
- payload = {
40
- caller: self,
41
- message: message,
42
- options: options,
43
- error: error,
44
- attempts_count: attempts_count
45
- }
46
-
47
- if attempts_count > WaterDrop.config.kafka.max_retries
48
- WaterDrop.monitor.instrument("#{scope}.error", payload)
49
- false
50
- else
51
- WaterDrop.monitor.instrument("#{scope}.retry", payload)
52
- true
53
- end
54
- end
55
- end
56
- end
57
- end
@@ -1,52 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- # Engine used to propagate config application to DeliveryBoy with corner case handling
5
- module ConfigApplier
6
- class << self
7
- # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
8
- # @param settings [Hash] hash with WaterDrop settings
9
- def call(delivery_boy_config, settings)
10
- # Recursive lambda for mapping config down to delivery boy
11
- settings.each do |key, value|
12
- call(delivery_boy_config, value) && next if value.is_a?(Hash)
13
-
14
- # If this is a special case that needs manual setup instead of a direct reassignment
15
- if respond_to?(key, true)
16
- send(key, delivery_boy_config, value)
17
- else
18
- # If this setting is our internal one, we should not sync it with the delivery boy
19
- next unless delivery_boy_config.respond_to?(:"#{key}=")
20
-
21
- delivery_boy_config.public_send(:"#{key}=", value)
22
- end
23
- end
24
- end
25
-
26
- private
27
-
28
- # Extra setup for the compression codec as it behaves differently than other settings
29
- # that are ported 1:1 from ruby-kafka
30
- # For some crazy reason, delivery boy requires compression codec as a string, when
31
- # ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
32
- # that by assigning a string version that down the road will be symbolized again
33
- # by delivery boy
34
- # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
35
- # @param codec_name [Symbol] codec name as a symbol
36
- def compression_codec(delivery_boy_config, codec_name)
37
- # If there is no compression codec, we don't apply anything
38
- return unless codec_name
39
-
40
- delivery_boy_config.compression_codec = codec_name.to_s
41
- end
42
-
43
- # We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
44
- # manually
45
- # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
46
- # @param seed_brokers [Array<String>] kafka seed brokers
47
- def seed_brokers(delivery_boy_config, seed_brokers)
48
- delivery_boy_config.brokers = seed_brokers
49
- end
50
- end
51
- end
52
- end
@@ -1,19 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- module WaterDrop
4
- module Contracts
5
- # Contract with validation rules for validating that all the message options that
6
- # we provide to producer ale valid and usable
7
- # @note Does not validate message itself as it is not our concern
8
- class MessageOptions < Dry::Validation::Contract
9
- params do
10
- required(:topic).filled(:str?, format?: TOPIC_REGEXP)
11
- optional(:key).maybe(:str?, :filled?)
12
- optional(:partition).filled(:int?, gteq?: 0)
13
- optional(:partition_key).maybe(:str?, :filled?)
14
- optional(:create_time).maybe(:time?)
15
- optional(:headers).maybe(:hash?)
16
- end
17
- end
18
- end
19
- end
@@ -1,24 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- # WaterDrop library
4
- module WaterDrop
5
- # Sync producer for messages
6
- class SyncProducer < BaseProducer
7
- # Performs message delivery using deliver_async method
8
- # @param message [String] message that we want to send to Kafka
9
- # @param options [Hash] options (including topic) for producer
10
- # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
11
- # somehow invalid and we cannot perform delivery because of that
12
- def self.call(message, options)
13
- attempts_count ||= 0
14
- attempts_count += 1
15
-
16
- validate!(options)
17
- return unless WaterDrop.config.deliver
18
-
19
- DeliveryBoy.deliver(message, **options)
20
- rescue Kafka::Error => e
21
- graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
22
- end
23
- end
24
- end