waterdrop 1.4.2 → 2.0.2

data/lib/water_drop/contracts/message.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  module Contracts
+    # Contract with validation rules for validating that all the message options that
+    # we provide to producer ale valid and usable
+    class Message < Dry::Validation::Contract
+      # Regex to check that topic has a valid format
+      TOPIC_REGEXP = /\A(\w|-|\.)+\z/.freeze
+
+      # Checks, that the given value is a string
+      STRING_ASSERTION = ->(value) { value.is_a?(String) }.to_proc
+
+      private_constant :TOPIC_REGEXP, :STRING_ASSERTION
+
+      config.messages.load_paths << File.join(WaterDrop.gem_root, 'config', 'errors.yml')
+
+      option :max_payload_size
+
+      params do
+        required(:topic).filled(:str?, format?: TOPIC_REGEXP)
+        required(:payload).filled(:str?)
+        optional(:key).maybe(:str?, :filled?)
+        optional(:partition).filled(:int?, gteq?: -1)
+        optional(:partition_key).maybe(:str?, :filled?)
+        optional(:timestamp).maybe { time? | int? }
+        optional(:headers).maybe(:hash?)
+      end
+
+      rule(:headers) do
+        next unless value.is_a?(Hash)
+
+        key.failure(:invalid_key_type) unless value.keys.all?(&STRING_ASSERTION)
+        key.failure(:invalid_value_type) unless value.values.all?(&STRING_ASSERTION)
+      end
+
+      rule(:payload) do
+        key.failure(:max_payload_size) if value.bytesize > max_payload_size
+      end
+    end
+  end
+end

data/lib/water_drop/errors.rb

@@ -7,12 +7,38 @@ module WaterDrop
     BaseError = Class.new(StandardError)
 
     # Raised when configuration doesn't match with validation contract
-    InvalidConfiguration = Class.new(BaseError)
+    ConfigurationInvalidError = Class.new(BaseError)
 
-    # Raised when we try to send message with invalid options
-    InvalidMessageOptions = Class.new(BaseError)
+    # Raised when we want to use a producer that was not configured
+    ProducerNotConfiguredError = Class.new(BaseError)
 
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    # Raised when we want to reconfigure a producer that was already configured
+    ProducerAlreadyConfiguredError = Class.new(BaseError)
+
+    # Raised when trying to use connected producer from a forked child process
+    # Producers cannot be used in forks if they were already used in the child processes
+    ProducerUsedInParentProcess = Class.new(BaseError)
+
+    # Raised when there was an attempt to use a closed producer
+    ProducerClosedError = Class.new(BaseError)
+
+    # Raised when we want to send a message that is invalid (impossible topic, etc)
+    MessageInvalidError = Class.new(BaseError)
+
+    # Raised when we've got an unexpected status. This should never happen. If it does, please
+    # contact us as it is an error.
+    StatusInvalidError = Class.new(BaseError)
+
+    # Raised when during messages flushing something bad happened
+    class FlushFailureError < BaseError
+      attr_reader :dispatched_messages
+
+      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+      #   messages that we've dispatched
+      def initialize(dispatched_messages)
+        super()
+        @dispatched_messages = dispatched_messages
+      end
+    end
   end
 end

data/lib/water_drop/instrumentation/monitor.rb

@@ -11,34 +11,28 @@ module WaterDrop
     class Monitor < Dry::Monitor::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
-      BASE_EVENTS = %w[
-        async_producer.call.error
-        async_producer.call.retry
-        sync_producer.call.error
-        sync_producer.call.retry
+      EVENTS = %w[
+        producer.closed
+        message.produced_async
+        message.produced_sync
+        messages.produced_async
+        messages.produced_sync
+        message.buffered
+        messages.buffered
+        message.acknowledged
+        buffer.flushed_async
+        buffer.flushed_async.error
+        buffer.flushed_sync
+        buffer.flushed_sync.error
+        statistics.emitted
       ].freeze
 
-      private_constant :BASE_EVENTS
+      private_constant :EVENTS
 
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super(:waterdrop)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+        EVENTS.each(&method(:register_event))
       end
     end
   end

data/lib/water_drop/instrumentation/stdout_listener.rb

@@ -7,38 +7,119 @@ module WaterDrop
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
     class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        info
-        error
-      ].freeze
-
-      %i[
-        sync_producer
-        async_producer
-      ].each do |producer_type|
-        error_name = :"on_#{producer_type}_call_error"
-        retry_name = :"on_#{producer_type}_call_retry"
-
-        define_method error_name do |event|
-          options = event[:options]
-          error = event[:error]
-          error "Delivery failure to: #{options} because of #{error}"
-        end
-
-        define_method retry_name do |event|
-          attempts_count = event[:attempts_count]
-          options = event[:options]
-          error = event[:error]
-
-          info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
-        end
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          WaterDrop.logger.send(log_level, *args)
-        end
+      # @param logger [Object] stdout logger we want to use
+      def initialize(logger)
+        @logger = logger
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_async(event)
+        message = event[:message]
+
+        info(event, "Async producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_sync(event)
+        message = event[:message]
+
+        info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_async(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_sync(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_buffered(event)
+        message = event[:message]
+
+        info(event, "Buffering of a message to '#{message[:topic]}' topic")
+        debug(event, [message, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_buffered(event)
+        messages = event[:messages]
+
+        info(event, "Buffering of #{messages.size} messages")
+        debug(event, [messages, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async(event)
+        messages = event[:messages]
+
+        info(event, "Async flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async_error(event)
+        messages = event[:messages]
+        error = event[:error]
+
+        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync(event)
+        messages = event[:messages]
+
+        info(event, "Sync flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync_error(event)
+        messages = event[:dispatched]
+        error = event[:error]
+
+        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closed(event)
+        info event, 'Closing producer'
+        debug event, event[:producer].messages.size
+      end
+
+      private
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def debug(event, log_message)
+        @logger.debug("[#{event[:producer].id}] #{log_message}")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def info(event, log_message)
+        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def error(event, log_message)
+        @logger.error("[#{event[:producer].id}] #{log_message}")
       end
     end
   end

data/lib/water_drop/patches/rdkafka_producer.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Patches to external components
+  module Patches
+    # `Rdkafka::Producer` patches
+    module RdkafkaProducer
+      # Errors upon which we want to retry message production
+      # @note Since production happens async, those errors should only occur when using
+      #   partition_key, thus only then we handle them
+      RETRYABLES = %w[
+        leader_not_available
+        err_not_leader_for_partition
+        invalid_replication_factor
+        transport
+        timed_out
+      ].freeze
+
+      # How many attempts do we want to make before re-raising the error
+      MAX_ATTEMPTS = 5
+
+      private_constant :RETRYABLES, :MAX_ATTEMPTS
+
+      # @param args [Object] anything `Rdkafka::Producer#produce` accepts
+      #
+      # @note This can be removed once this: https://github.com/appsignal/rdkafka-ruby/issues/163
+      #   is resolved.
+      def produce(**args)
+        attempt ||= 0
+        attempt += 1
+
+        super
+      rescue Rdkafka::RdkafkaError => e
+        raise unless args.key?(:partition_key)
+        # We care only about specific errors
+        # https://docs.confluent.io/platform/current/clients/librdkafka/html/md_INTRODUCTION.html
+        raise unless RETRYABLES.any? { |message| e.message.to_s.include?(message) }
+        raise if attempt > MAX_ATTEMPTS
+
+        max_sleep = 2**attempt / 10.0
+        sleep rand(0.01..max_sleep)
+
+        retry
+      end
+    end
+  end
+end
+
+Rdkafka::Producer.prepend(WaterDrop::Patches::RdkafkaProducer)

data/lib/water_drop/producer.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Main WaterDrop messages producer
+  class Producer
+    include Sync
+    include Async
+    include Buffer
+
+    # @return [String] uuid of the current producer
+    attr_reader :id
+    # @return [Status] producer status object
+    attr_reader :status
+    # @return [Concurrent::Array] internal messages buffer
+    attr_reader :messages
+    # @return [Object] monitor we want to use
+    attr_reader :monitor
+    # @return [Object] dry-configurable config object
+    attr_reader :config
+
+    # Creates a not-yet-configured instance of the producer
+    # @param block [Proc] configuration block
+    # @return [Producer] producer instance
+    def initialize(&block)
+      @buffer_mutex = Mutex.new
+      @connecting_mutex = Mutex.new
+      @closing_mutex = Mutex.new
+
+      @status = Status.new
+      @messages = Concurrent::Array.new
+
+      return unless block
+
+      setup(&block)
+    end
+
+    # Sets up the whole configuration and initializes all that is needed
+    # @param block [Block] configuration block
+    def setup(&block)
+      raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
+
+      @config = Config
+                .new
+                .setup(&block)
+                .config
+
+      @id = @config.id
+      @monitor = @config.monitor
+      @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
+      @status.configured!
+    end
+
+    # @return [Rdkafka::Producer] raw rdkafka producer
+    # @note Client is lazy initialized, keeping in mind also the fact of a potential fork that
+    #   can happen any time.
+    # @note It is not recommended to fork a producer that is already in use so in case of
+    #   bootstrapping a cluster, it's much better to fork configured but not used producers
+    def client
+      return @client if @client && @pid == Process.pid
+
+      # Don't allow to obtain a client reference for a producer that was not configured
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+
+      @connecting_mutex.synchronize do
+        return @client if @client && @pid == Process.pid
+
+        # We should raise an error when trying to use a producer from a fork, that is already
+        # connected to Kafka. We allow forking producers only before they are used
+        raise Errors::ProducerUsedInParentProcess, Process.pid if @status.connected?
+
+        # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
+        # process don't leak
+        ObjectSpace.undefine_finalizer(id)
+        # Finalizer tracking is needed for handling shutdowns gracefully.
+        # I don't expect everyone to remember about closing all the producers all the time, thus
+        # this approach is better. Although it is still worth keeping in mind, that this will
+        # block GC from removing a no longer used producer unless closed properly but at least
+        # won't crash the VM upon closing the process
+        ObjectSpace.define_finalizer(id, proc { close })
+
+        @pid = Process.pid
+        @client = Builder.new.call(self, @config)
+        @status.connected!
+      end
+
+      @client
+    end
+
+    # Flushes the buffers in a sync way and closes the producer
+    def close
+      @closing_mutex.synchronize do
+        return unless @status.active?
+
+        @monitor.instrument(
+          'producer.closed',
+          producer: self
+        ) do
+          @status.closing!
+
+          # No need for auto-gc if everything got closed by us
+          # This should be used only in case a producer was not closed properly and forgotten
+          ObjectSpace.undefine_finalizer(id)
+
+          # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
+          # as we close the client after the flushing (even if blocked by the mutex)
+          flush(true)
+
+          # We should not close the client in several threads the same time
+          # It is safe to run it several times but not exactly the same moment
+          client.close
+
+          @status.closed!
+        end
+      end
+    end
+
+    # Ensures that we don't run any operations when the producer is not configured or when it
+    # was already closed
+    def ensure_active!
+      return if @status.active?
+
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+
+      # This should never happen
+      raise Errors::StatusInvalidError, [id, @status.to_s]
+    end
+
+    # Ensures that the message we want to send out to Kafka is actually valid and that it can be
+    # sent there
+    # @param message [Hash] message we want to send
+    # @raise [Karafka::Errors::MessageInvalidError]
+    def validate_message!(message)
+      result = @contract.call(message)
+      return if result.success?
+
+      raise Errors::MessageInvalidError, [
+        result.errors.to_h,
+        message
+      ]
+    end
+  end
+end