waterdrop 1.4.2 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data.tar.gz.sig +0 -0
  4. data/.github/workflows/ci.yml +1 -2
  5. data/.gitignore +2 -0
  6. data/.ruby-version +1 -1
  7. data/CHANGELOG.md +17 -5
  8. data/Gemfile +9 -0
  9. data/Gemfile.lock +42 -29
  10. data/{MIT-LICENCE → MIT-LICENSE} +0 -0
  11. data/README.md +244 -57
  12. data/certs/mensfeld.pem +21 -21
  13. data/config/errors.yml +3 -16
  14. data/docker-compose.yml +1 -1
  15. data/lib/water_drop.rb +4 -24
  16. data/lib/water_drop/config.rb +41 -142
  17. data/lib/water_drop/contracts.rb +0 -2
  18. data/lib/water_drop/contracts/config.rb +8 -121
  19. data/lib/water_drop/contracts/message.rb +42 -0
  20. data/lib/water_drop/errors.rb +31 -5
  21. data/lib/water_drop/instrumentation/monitor.rb +16 -22
  22. data/lib/water_drop/instrumentation/stdout_listener.rb +113 -32
  23. data/lib/water_drop/patches/rdkafka_producer.rb +49 -0
  24. data/lib/water_drop/producer.rb +143 -0
  25. data/lib/water_drop/producer/async.rb +51 -0
  26. data/lib/water_drop/producer/buffer.rb +113 -0
  27. data/lib/water_drop/producer/builder.rb +63 -0
  28. data/lib/water_drop/producer/dummy_client.rb +32 -0
  29. data/lib/water_drop/producer/statistics_decorator.rb +71 -0
  30. data/lib/water_drop/producer/status.rb +52 -0
  31. data/lib/water_drop/producer/sync.rb +65 -0
  32. data/lib/water_drop/version.rb +1 -1
  33. data/waterdrop.gemspec +4 -4
  34. metadata +44 -45
  35. metadata.gz.sig +0 -0
  36. data/lib/water_drop/async_producer.rb +0 -26
  37. data/lib/water_drop/base_producer.rb +0 -57
  38. data/lib/water_drop/config_applier.rb +0 -52
  39. data/lib/water_drop/contracts/message_options.rb +0 -19
  40. data/lib/water_drop/sync_producer.rb +0 -24
@@ -0,0 +1,42 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WaterDrop
4
+ module Contracts
5
+ # Contract with validation rules for validating that all the message options that
6
+ # we provide to producer ale valid and usable
7
+ class Message < Dry::Validation::Contract
8
+ # Regex to check that topic has a valid format
9
+ TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
10
+
11
+ # Checks, that the given value is a string
12
+ STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
13
+
14
+ private_constant :TOPIC_REGEXP, :STRING_ASSERTION
15
+
16
+ config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
17
+
18
+ option :max_payload_size
19
+
20
+ params do
21
+ required(:topic).filled(:str?, format?: TOPIC_REGEXP)
22
+ required(:payload).filled(:str?)
23
+ optional(:key).maybe(:str?, :filled?)
24
+ optional(:partition).filled(:int?, gteq?: -1)
25
+ optional(:partition_key).maybe(:str?, :filled?)
26
+ optional(:timestamp).maybe { time? | int? }
27
+ optional(:headers).maybe(:hash?)
28
+ end
29
+
30
+ rule(:headers) do
31
+ next unless value.is_a?(Hash)
32
+
33
+ key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
34
+ key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
35
+ end
36
+
37
+ rule(:payload) do
38
+ key.failure(:max_payload_size) if value.bytesize > max_payload_size
39
+ end
40
+ end
41
+ end
42
+ end
@@ -7,12 +7,38 @@ module WaterDrop
7
7
  BaseError = Class.new(StandardError)
8
8
 
9
9
  # Raised when configuration doesn't match with validation contract
10
- InvalidConfiguration = Class.new(BaseError)
10
+ ConfigurationInvalidError = Class.new(BaseError)
11
11
 
12
- # Raised when we try to send message with invalid options
13
- InvalidMessageOptions = Class.new(BaseError)
12
+ # Raised when we want to use a producer that was not configured
13
+ ProducerNotConfiguredError = Class.new(BaseError)
14
14
 
15
- # Raised when want to hook up to an event that is not registered and supported
16
- UnregisteredMonitorEvent = Class.new(BaseError)
15
+ # Raised when we want to reconfigure a producer that was already configured
16
+ ProducerAlreadyConfiguredError = Class.new(BaseError)
17
+
18
+ # Raised when trying to use connected producer from a forked child process
19
+ # Producers cannot be used in forks if they were already used in the child processes
20
+ ProducerUsedInParentProcess = Class.new(BaseError)
21
+
22
+ # Raised when there was an attempt to use a closed producer
23
+ ProducerClosedError = Class.new(BaseError)
24
+
25
+ # Raised when we want to send a message that is invalid (impossible topic, etc)
26
+ MessageInvalidError = Class.new(BaseError)
27
+
28
+ # Raised when we've got an unexpected status. This should never happen. If it does, please
29
+ # contact us as it is an error.
30
+ StatusInvalidError = Class.new(BaseError)
31
+
32
+ # Raised when during messages flushing something bad happened
33
+ class FlushFailureError < BaseError
34
+ attr_reader :dispatched_messages
35
+
36
+ # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
37
+ # messages that we've dispatched
38
+ def initialize(dispatched_messages)
39
+ super()
40
+ @dispatched_messages = dispatched_messages
41
+ end
42
+ end
17
43
  end
18
44
  end
@@ -11,34 +11,28 @@ module WaterDrop
11
11
  class Monitor < Dry::Monitor::Notifications
12
12
  # List of events that we support in the system and to which a monitor client can hook up
13
13
  # @note The non-error once support timestamp benchmarking
14
- BASE_EVENTS = %w[
15
- async_producer.call.error
16
- async_producer.call.retry
17
- sync_producer.call.error
18
- sync_producer.call.retry
14
+ EVENTS = %w[
15
+ producer.closed
16
+ message.produced_async
17
+ message.produced_sync
18
+ messages.produced_async
19
+ messages.produced_sync
20
+ message.buffered
21
+ messages.buffered
22
+ message.acknowledged
23
+ buffer.flushed_async
24
+ buffer.flushed_async.error
25
+ buffer.flushed_sync
26
+ buffer.flushed_sync.error
27
+ statistics.emitted
19
28
  ].freeze
20
29
 
21
- private_constant :BASE_EVENTS
30
+ private_constant :EVENTS
22
31
 
23
32
  # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
24
33
  def initialize
25
34
  super(:waterdrop)
26
- BASE_EVENTS.each(&method(:register_event))
27
- end
28
-
29
- # Allows us to subscribe to events with a code that will be yielded upon events
30
- # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
31
- # or a listener if we decide to go with object listener
32
- def subscribe(event_name_or_listener)
33
- return super unless event_name_or_listener.is_a?(String)
34
- return super if available_events.include?(event_name_or_listener)
35
-
36
- raise Errors::UnregisteredMonitorEvent, event_name_or_listener
37
- end
38
-
39
- # @return [Array<String>] names of available events to which we can subscribe
40
- def available_events
41
- __bus__.events.keys
35
+ EVENTS.each(&method(:register_event))
42
36
  end
43
37
  end
44
38
  end
@@ -7,38 +7,119 @@ module WaterDrop
7
7
  # @note It is a module as we can use it then as a part of the Karafka framework listener
8
8
  # as well as we can use it standalone
9
9
  class StdoutListener
10
- # Log levels that we use in this particular listener
11
- USED_LOG_LEVELS = %i[
12
- info
13
- error
14
- ].freeze
15
-
16
- %i[
17
- sync_producer
18
- async_producer
19
- ].each do |producer_type|
20
- error_name = :"on_#{producer_type}_call_error"
21
- retry_name = :"on_#{producer_type}_call_retry"
22
-
23
- define_method error_name do |event|
24
- options = event[:options]
25
- error = event[:error]
26
- error "Delivery failure to: #{options} because of #{error}"
27
- end
28
-
29
- define_method retry_name do |event|
30
- attempts_count = event[:attempts_count]
31
- options = event[:options]
32
- error = event[:error]
33
-
34
- info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
35
- end
36
- end
37
-
38
- USED_LOG_LEVELS.each do |log_level|
39
- define_method log_level do |*args|
40
- WaterDrop.logger.send(log_level, *args)
41
- end
10
+ # @param logger [Object] stdout logger we want to use
11
+ def initialize(logger)
12
+ @logger = logger
13
+ end
14
+
15
+ # @param event [Dry::Events::Event] event that happened with the details
16
+ def on_message_produced_async(event)
17
+ message = event[:message]
18
+
19
+ info(event, "Async producing of a message to '#{message[:topic]}' topic")
20
+ debug(event, message)
21
+ end
22
+
23
+ # @param event [Dry::Events::Event] event that happened with the details
24
+ def on_message_produced_sync(event)
25
+ message = event[:message]
26
+
27
+ info(event, "Sync producing of a message to '#{message[:topic]}' topic")
28
+ debug(event, message)
29
+ end
30
+
31
+ # @param event [Dry::Events::Event] event that happened with the details
32
+ def on_messages_produced_async(event)
33
+ messages = event[:messages]
34
+ topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
35
+
36
+ info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
37
+ debug(event, messages)
38
+ end
39
+
40
+ # @param event [Dry::Events::Event] event that happened with the details
41
+ def on_messages_produced_sync(event)
42
+ messages = event[:messages]
43
+ topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
44
+
45
+ info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
46
+ debug(event, messages)
47
+ end
48
+
49
+ # @param event [Dry::Events::Event] event that happened with the details
50
+ def on_message_buffered(event)
51
+ message = event[:message]
52
+
53
+ info(event, "Buffering of a message to '#{message[:topic]}' topic")
54
+ debug(event, [message, event[:producer].messages.size])
55
+ end
56
+
57
+ # @param event [Dry::Events::Event] event that happened with the details
58
+ def on_messages_buffered(event)
59
+ messages = event[:messages]
60
+
61
+ info(event, "Buffering of #{messages.size} messages")
62
+ debug(event, [messages, event[:producer].messages.size])
63
+ end
64
+
65
+ # @param event [Dry::Events::Event] event that happened with the details
66
+ def on_buffer_flushed_async(event)
67
+ messages = event[:messages]
68
+
69
+ info(event, "Async flushing of #{messages.size} messages from the buffer")
70
+ debug(event, messages)
71
+ end
72
+
73
+ # @param event [Dry::Events::Event] event that happened with the details
74
+ def on_buffer_flushed_async_error(event)
75
+ messages = event[:messages]
76
+ error = event[:error]
77
+
78
+ error(event, "Async flushing of #{messages.size} failed due to: #{error}")
79
+ debug(event, messages)
80
+ end
81
+
82
+ # @param event [Dry::Events::Event] event that happened with the details
83
+ def on_buffer_flushed_sync(event)
84
+ messages = event[:messages]
85
+
86
+ info(event, "Sync flushing of #{messages.size} messages from the buffer")
87
+ debug(event, messages)
88
+ end
89
+
90
+ # @param event [Dry::Events::Event] event that happened with the details
91
+ def on_buffer_flushed_sync_error(event)
92
+ messages = event[:dispatched]
93
+ error = event[:error]
94
+
95
+ error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
96
+ debug(event, messages)
97
+ end
98
+
99
+ # @param event [Dry::Events::Event] event that happened with the details
100
+ def on_producer_closed(event)
101
+ info event, 'Closing producer'
102
+ debug event, event[:producer].messages.size
103
+ end
104
+
105
+ private
106
+
107
+ # @param event [Dry::Events::Event] event that happened with the details
108
+ # @param log_message [String] message we want to publish
109
+ def debug(event, log_message)
110
+ @logger.debug("[#{event[:producer].id}] #{log_message}")
111
+ end
112
+
113
+ # @param event [Dry::Events::Event] event that happened with the details
114
+ # @param log_message [String] message we want to publish
115
+ def info(event, log_message)
116
+ @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
117
+ end
118
+
119
+ # @param event [Dry::Events::Event] event that happened with the details
120
+ # @param log_message [String] message we want to publish
121
+ def error(event, log_message)
122
+ @logger.error("[#{event[:producer].id}] #{log_message}")
42
123
  end
43
124
  end
44
125
  end
@@ -0,0 +1,49 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WaterDrop
4
+ # Patches to external components
5
+ module Patches
6
+ # `Rdkafka::Producer` patches
7
+ module RdkafkaProducer
8
+ # Errors upon which we want to retry message production
9
+ # @note Since production happens async, those errors should only occur when using
10
+ # partition_key, thus only then we handle them
11
+ RETRYABLES = %w[
12
+ leader_not_available
13
+ err_not_leader_for_partition
14
+ invalid_replication_factor
15
+ transport
16
+ timed_out
17
+ ].freeze
18
+
19
+ # How many attempts do we want to make before re-raising the error
20
+ MAX_ATTEMPTS = 5
21
+
22
+ private_constant :RETRYABLES, :MAX_ATTEMPTS
23
+
24
+ # @param args [Object] anything `Rdkafka::Producer#produce` accepts
25
+ #
26
+ # @note This can be removed once this: https://github.com/appsignal/rdkafka-ruby/issues/163
27
+ # is resolved.
28
+ def produce(**args)
29
+ attempt ||= 0
30
+ attempt += 1
31
+
32
+ super
33
+ rescue Rdkafka::RdkafkaError => e
34
+ raise unless args.key?(:partition_key)
35
+ # We care only about specific errors
36
+ # https://docs.confluent.io/platform/current/clients/librdkafka/html/md_INTRODUCTION.html
37
+ raise unless RETRYABLES.any? { |message| e.message.to_s.include?(message) }
38
+ raise if attempt > MAX_ATTEMPTS
39
+
40
+ max_sleep = 2**attempt / 10.0
41
+ sleep rand(0.01..max_sleep)
42
+
43
+ retry
44
+ end
45
+ end
46
+ end
47
+ end
48
+
49
+ Rdkafka::Producer.prepend(WaterDrop::Patches::RdkafkaProducer)
@@ -0,0 +1,143 @@
1
+ # frozen_string_literal: true
2
+
3
+ module WaterDrop
4
+ # Main WaterDrop messages producer
5
+ class Producer
6
+ include Sync
7
+ include Async
8
+ include Buffer
9
+
10
+ # @return [String] uuid of the current producer
11
+ attr_reader :id
12
+ # @return [Status] producer status object
13
+ attr_reader :status
14
+ # @return [Concurrent::Array] internal messages buffer
15
+ attr_reader :messages
16
+ # @return [Object] monitor we want to use
17
+ attr_reader :monitor
18
+ # @return [Object] dry-configurable config object
19
+ attr_reader :config
20
+
21
+ # Creates a not-yet-configured instance of the producer
22
+ # @param block [Proc] configuration block
23
+ # @return [Producer] producer instance
24
+ def initialize(&block)
25
+ @buffer_mutex = Mutex.new
26
+ @connecting_mutex = Mutex.new
27
+ @closing_mutex = Mutex.new
28
+
29
+ @status = Status.new
30
+ @messages = Concurrent::Array.new
31
+
32
+ return unless block
33
+
34
+ setup(&block)
35
+ end
36
+
37
+ # Sets up the whole configuration and initializes all that is needed
38
+ # @param block [Block] configuration block
39
+ def setup(&block)
40
+ raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
41
+
42
+ @config = Config
43
+ .new
44
+ .setup(&block)
45
+ .config
46
+
47
+ @id = @config.id
48
+ @monitor = @config.monitor
49
+ @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
50
+ @status.configured!
51
+ end
52
+
53
+ # @return [Rdkafka::Producer] raw rdkafka producer
54
+ # @note Client is lazy initialized, keeping in mind also the fact of a potential fork that
55
+ # can happen any time.
56
+ # @note It is not recommended to fork a producer that is already in use so in case of
57
+ # bootstrapping a cluster, it's much better to fork configured but not used producers
58
+ def client
59
+ return @client if @client && @pid == Process.pid
60
+
61
+ # Don't allow to obtain a client reference for a producer that was not configured
62
+ raise Errors::ProducerNotConfiguredError, id if @status.initial?
63
+
64
+ @connecting_mutex.synchronize do
65
+ return @client if @client && @pid == Process.pid
66
+
67
+ # We should raise an error when trying to use a producer from a fork, that is already
68
+ # connected to Kafka. We allow forking producers only before they are used
69
+ raise Errors::ProducerUsedInParentProcess, Process.pid if @status.connected?
70
+
71
+ # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
72
+ # process don't leak
73
+ ObjectSpace.undefine_finalizer(id)
74
+ # Finalizer tracking is needed for handling shutdowns gracefully.
75
+ # I don't expect everyone to remember about closing all the producers all the time, thus
76
+ # this approach is better. Although it is still worth keeping in mind, that this will
77
+ # block GC from removing a no longer used producer unless closed properly but at least
78
+ # won't crash the VM upon closing the process
79
+ ObjectSpace.define_finalizer(id, proc { close })
80
+
81
+ @pid = Process.pid
82
+ @client = Builder.new.call(self, @config)
83
+ @status.connected!
84
+ end
85
+
86
+ @client
87
+ end
88
+
89
+ # Flushes the buffers in a sync way and closes the producer
90
+ def close
91
+ @closing_mutex.synchronize do
92
+ return unless @status.active?
93
+
94
+ @monitor.instrument(
95
+ 'producer.closed',
96
+ producer: self
97
+ ) do
98
+ @status.closing!
99
+
100
+ # No need for auto-gc if everything got closed by us
101
+ # This should be used only in case a producer was not closed properly and forgotten
102
+ ObjectSpace.undefine_finalizer(id)
103
+
104
+ # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
105
+ # as we close the client after the flushing (even if blocked by the mutex)
106
+ flush(true)
107
+
108
+ # We should not close the client in several threads the same time
109
+ # It is safe to run it several times but not exactly the same moment
110
+ client.close
111
+
112
+ @status.closed!
113
+ end
114
+ end
115
+ end
116
+
117
+ # Ensures that we don't run any operations when the producer is not configured or when it
118
+ # was already closed
119
+ def ensure_active!
120
+ return if @status.active?
121
+
122
+ raise Errors::ProducerNotConfiguredError, id if @status.initial?
123
+ raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
124
+
125
+ # This should never happen
126
+ raise Errors::StatusInvalidError, [id, @status.to_s]
127
+ end
128
+
129
+ # Ensures that the message we want to send out to Kafka is actually valid and that it can be
130
+ # sent there
131
+ # @param message [Hash] message we want to send
132
+ # @raise [Karafka::Errors::MessageInvalidError]
133
+ def validate_message!(message)
134
+ result = @contract.call(message)
135
+ return if result.success?
136
+
137
+ raise Errors::MessageInvalidError, [
138
+ result.errors.to_h,
139
+ message
140
+ ]
141
+ end
142
+ end
143
+ end