waterdrop 1.4.0 → 2.0.0.rc1

data/lib/water_drop/errors.rb

@@ -7,12 +7,37 @@ module WaterDrop
     BaseError = Class.new(StandardError)
 
     # Raised when configuration doesn't match with validation contract
-    InvalidConfiguration = Class.new(BaseError)
+    ConfigurationInvalidError = Class.new(BaseError)
 
-    # Raised when we try to send message with invalid options
-    InvalidMessageOptions = Class.new(BaseError)
+    # Raised when we want to use a producer that was not configured
+    ProducerNotConfiguredError = Class.new(BaseError)
 
-    # Raised when want to hook up to an event that is not registered and supported
-    UnregisteredMonitorEvent = Class.new(BaseError)
+    # Raised when we want to reconfigure a producer that was already configured
+    ProducerAlreadyConfiguredError = Class.new(BaseError)
+
+    # Raised when trying to use connected producer from a forked child process
+    # Producers cannot be used in forks if they were already used in the child processes
+    ProducerUsedInParentProcess = Class.new(BaseError)
+
+    # Raised when there was an attempt to use a closed producer
+    ProducerClosedError = Class.new(BaseError)
+
+    # Raised when we want to send a message that is invalid (impossible topic, etc)
+    MessageInvalidError = Class.new(BaseError)
+
+    # Raised when we've got an unexpected status. This should never happen. If it does, please
+    # contact us as it is an error.
+    StatusInvalidError = Class.new(BaseError)
+
+    # Raised when during messages flushing something bad happened
+    class FlushFailureError < BaseError
+      attr_reader :dispatched_messages
+
+      # @param dispatched_messages [Array<Rdkafka::Producer::DeliveryHandle>] handlers of the
+      #   messages that we've dispatched
+      def initialize(dispatched_messages)
+        @dispatched_messages = dispatched_messages
+      end
+    end
   end
 end

data/lib/water_drop/instrumentation.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Namespace for all the things related with WaterDrop instrumentation process
+  module Instrumentation
+  end
+end

data/lib/water_drop/instrumentation/monitor.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 module WaterDrop
-  # Namespace for all the things related with WaterDrop instrumentation process
   module Instrumentation
     # Monitor is used to hookup external monitoring services to monitor how WaterDrop works
     # Since it is a pub-sub based on dry-monitor, you can use as many subscribers/loggers at the
@@ -12,34 +11,28 @@ module WaterDrop
     class Monitor < Dry::Monitor::Notifications
       # List of events that we support in the system and to which a monitor client can hook up
       # @note The non-error once support timestamp benchmarking
-      BASE_EVENTS = %w[
-        async_producer.call.error
-        async_producer.call.retry
-        sync_producer.call.error
-        sync_producer.call.retry
+      EVENTS = %w[
+        producer.closed
+        message.produced_async
+        message.produced_sync
+        messages.produced_async
+        messages.produced_sync
+        message.buffered
+        messages.buffered
+        message.acknowledged
+        buffer.flushed_async
+        buffer.flushed_async.error
+        buffer.flushed_sync
+        buffer.flushed_sync.error
+        statistics.emitted
       ].freeze
 
-      private_constant :BASE_EVENTS
+      private_constant :EVENTS
 
       # @return [WaterDrop::Instrumentation::Monitor] monitor instance for system instrumentation
       def initialize
         super(:waterdrop)
-        BASE_EVENTS.each(&method(:register_event))
-      end
-
-      # Allows us to subscribe to events with a code that will be yielded upon events
-      # @param event_name_or_listener [String, Object] name of the event we want to subscribe to
-      #   or a listener if we decide to go with object listener
-      def subscribe(event_name_or_listener)
-        return super unless event_name_or_listener.is_a?(String)
-        return super if available_events.include?(event_name_or_listener)
-
-        raise Errors::UnregisteredMonitorEvent, event_name_or_listener
-      end
-
-      # @return [Array<String>] names of available events to which we can subscribe
-      def available_events
-        __bus__.events.keys
+        EVENTS.each(&method(:register_event))
       end
     end
   end

data/lib/water_drop/instrumentation/stdout_listener.rb

@@ -7,38 +7,119 @@ module WaterDrop
     # @note It is a module as we can use it then as a part of the Karafka framework listener
     #   as well as we can use it standalone
     class StdoutListener
-      # Log levels that we use in this particular listener
-      USED_LOG_LEVELS = %i[
-        info
-        error
-      ].freeze
-
-      %i[
-        sync_producer
-        async_producer
-      ].each do |producer_type|
-        error_name = :"on_#{producer_type}_call_error"
-        retry_name = :"on_#{producer_type}_call_retry"
-
-        define_method error_name do |event|
-          options = event[:options]
-          error = event[:error]
-          error "Delivery failure to: #{options} because of #{error}"
-        end
-
-        define_method retry_name do |event|
-          attempts_count = event[:attempts_count]
-          options = event[:options]
-          error = event[:error]
-
-          info "Attempt #{attempts_count} of delivery to: #{options} because of #{error}"
-        end
-      end
-
-      USED_LOG_LEVELS.each do |log_level|
-        define_method log_level do |*args|
-          WaterDrop.logger.send(log_level, *args)
-        end
+      # @param logger [Object] stdout logger we want to use
+      def initialize(logger)
+        @logger = logger
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_async(event)
+        message = event[:message]
+
+        info(event, "Async producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_produced_sync(event)
+        message = event[:message]
+
+        info(event, "Sync producing of a message to '#{message[:topic]}' topic")
+        debug(event, message)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_async(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Async producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_produced_sync(event)
+        messages = event[:messages]
+        topics_count = messages.map { |message| "'#{message[:topic]}'" }.uniq.count
+
+        info(event, "Sync producing of #{messages.size} messages to #{topics_count} topics")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_message_buffered(event)
+        message = event[:message]
+
+        info(event, "Buffering of a message to '#{message[:topic]}' topic")
+        debug(event, [message, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_messages_buffered(event)
+        messages = event[:messages]
+
+        info(event, "Buffering of #{messages.size} messages")
+        debug(event, [messages, event[:producer].messages.size])
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async(event)
+        messages = event[:messages]
+
+        info(event, "Async flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_async_error(event)
+        messages = event[:messages]
+        error = event[:error]
+
+        error(event, "Async flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync(event)
+        messages = event[:messages]
+
+        info(event, "Sync flushing of #{messages.size} messages from the buffer")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_buffer_flushed_sync_error(event)
+        messages = event[:dispatched]
+        error = event[:error]
+
+        error(event, "Sync flushing of #{messages.size} failed due to: #{error}")
+        debug(event, messages)
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_producer_closed(event)
+        info event, 'Closing producer'
+        debug event, event[:producer].messages.size
+      end
+
+      private
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def debug(event, log_message)
+        @logger.debug("[#{event[:producer].id}] #{log_message}")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def info(event, log_message)
+        @logger.info("[#{event[:producer].id}] #{log_message} took #{event[:time]} ms")
+      end
+
+      # @param event [Dry::Events::Event] event that happened with the details
+      # @param log_message [String] message we want to publish
+      def error(event, log_message)
+        @logger.error("[#{event[:producer].id}] #{log_message}")
       end
     end
   end

data/lib/water_drop/producer.rb

@@ -0,0 +1,142 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Main WaterDrop messages producer
+  class Producer
+    include Sync
+    include Async
+    include Buffer
+
+    # @return [String] uuid of the current producer
+    attr_reader :id
+    # @return [Status] producer status object
+    attr_reader :status
+    # @return [Concurrent::Array] internal messages buffer
+    attr_reader :messages
+    # @return [Object] monitor we want to use
+    attr_reader :monitor
+    # @return [Object] dry-configurable config object
+    attr_reader :config
+
+    # Creates a not-yet-configured instance of the producer
+    # @param block [Proc] configuration block
+    # @return [Producer] producer instance
+    def initialize(&block)
+      @buffer_mutex = Mutex.new
+      @connecting_mutex = Mutex.new
+      @closing_mutex = Mutex.new
+
+      @status = Status.new
+      @messages = Concurrent::Array.new
+
+      return unless block
+
+      setup(&block)
+    end
+
+    # Sets up the whole configuration and initializes all that is needed
+    # @param block [Block] configuration block
+    def setup(&block)
+      raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
+
+      @config = Config
+                .new
+                .setup(&block)
+                .config
+
+      @id = @config.id
+      @monitor = @config.monitor
+      @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
+      @status.configured!
+    end
+
+    # @return [Rdkafka::Producer] raw rdkafka producer
+    # @note Client is lazy initialized, keeping in mind also the fact of a potential fork that
+    #   can happen any time.
+    # @note It is not recommended to fork a producer that is already in use so in case of
+    #   bootstrapping a cluster, it's much better to fork configured but not used producers
+    def client
+      return @client if @client && @pid == Process.pid
+
+      # Don't allow to obtain a client reference for a producer that was not configured
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+
+      @connecting_mutex.synchronize do
+        return @client if @client && @pid == Process.pid
+
+        # We should raise an error when trying to use a producer from a fork, that is already
+        # connected to Kafka. We allow forking producers only before they are used
+        raise Errors::ProducerUsedInParentProcess, Process.pid if @status.connected?
+
+        # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
+        # process don't leak
+        ObjectSpace.undefine_finalizer(self)
+        # Finalizer tracking is needed for handling shutdowns gracefully.
+        # I don't expect everyone to remember about closing all the producers all the time, thus
+        # this approach is better. Although it is still worth keeping in mind, that this will
+        # block GC from removing a no longer used producer unless closed properly
+        ObjectSpace.define_finalizer(self, proc { close })
+
+        @pid = Process.pid
+        @client = Builder.new.call(self, @config)
+        @status.connected!
+      end
+
+      @client
+    end
+
+    # Flushes the buffers in a sync way and closes the producer
+    def close
+      @closing_mutex.synchronize do
+        return unless @status.active?
+
+        @monitor.instrument(
+          'producer.closed',
+          producer: self
+        ) do
+          @status.closing!
+
+          # No need for auto-gc if everything got closed by us
+          # This should be used only in case a producer was not closed properly and forgotten
+          ObjectSpace.undefine_finalizer(self)
+
+          # Flush has it's own buffer mutex but even if it is blocked, flushing can still happen
+          # as we close the client after the flushing (even if blocked by the mutex)
+          flush(false)
+
+          # We should not close the client in several threads the same time
+          # It is safe to run it several times but not exactly the same moment
+          client.close
+
+          @status.closed!
+        end
+      end
+    end
+
+    # Ensures that we don't run any operations when the producer is not configured or when it
+    # was already closed
+    def ensure_active!
+      return if @status.active?
+
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+
+      # This should never happen
+      raise Errors::StatusInvalidError, [id, @status.to_s]
+    end
+
+    # Ensures that the message we want to send out to Kafka is actually valid and that it can be
+    # sent there
+    # @param message [Hash] message we want to send
+    # @raise [Karafka::Errors::MessageInvalidError]
+    def validate_message!(message)
+      result = @contract.call(message)
+      return if result.success?
+
+      raise Errors::MessageInvalidError, [
+        result.errors.to_h,
+        message
+      ]
+    end
+  end
+end

data/lib/water_drop/producer/async.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for asynchronous producer operations
+    module Async
+      # Produces a message to Kafka and does not wait for results
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryHandle] delivery handle that might return the report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_async(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_async',
+          producer: self,
+          message: message
+        ) { client.produce(**message) }
+      end
+
+      # Produces many messages to Kafka and does not wait for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_async(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.produced_async',
+          producer: self,
+          messages: messages
+        ) do
+          messages.map { |message| client.produce(**message) }
+        end
+      end
+    end
+  end
+end