waterdrop 1.4.0 → 2.0.1

data/lib/water_drop/producer.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Main WaterDrop messages producer
+  class Producer
+    include Sync
+    include Async
+    include Buffer
+
+    # @return [String] uuid of the current producer
+    attr_reader :id
+    # @return [Status] producer status object
+    attr_reader :status
+    # @return [Concurrent::Array] internal messages buffer
+    attr_reader :messages
+    # @return [Object] monitor we want to use
+    attr_reader :monitor
+    # @return [Object] dry-configurable config object
+    attr_reader :config
+
+    # Creates a not-yet-configured instance of the producer
+    # @param block [Proc] configuration block
+    # @return [Producer] producer instance
+    def initialize(&block)
+      @buffer_mutex = Mutex.new
+      @connecting_mutex = Mutex.new
+      @closing_mutex = Mutex.new
+
+      @status = Status.new
+      @messages = Concurrent::Array.new
+
+      return unless block
+
+      setup(&block)
+    end
+
+    # Sets up the whole configuration and initializes all that is needed
+    # @param block [Block] configuration block
+    def setup(&block)
+      raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
+
+      @config = Config
+                .new
+                .setup(&block)
+                .config
+
+      @id = @config.id
+      @monitor = @config.monitor
+      @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
+      @status.configured!
+    end
+
+    # @return [Rdkafka::Producer] raw rdkafka producer
+    # @note Client is lazy initialized, keeping in mind also the fact of a potential fork that
+    #   can happen any time.
+    # @note It is not recommended to fork a producer that is already in use so in case of
+    #   bootstrapping a cluster, it's much better to fork configured but not used producers
+    def client
+      return @client if @client && @pid == Process.pid
+
+      # Don't allow to obtain a client reference for a producer that was not configured
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+
+      @connecting_mutex.synchronize do
+        return @client if @client && @pid == Process.pid
+
+        # We should raise an error when trying to use a producer from a fork, that is already
+        # connected to Kafka. We allow forking producers only before they are used
+        raise Errors::ProducerUsedInParentProcess, Process.pid if @status.connected?
+
+        # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
+        # process don't leak
+        ObjectSpace.undefine_finalizer(id)
+        # Finalizer tracking is needed for handling shutdowns gracefully.
+        # I don't expect everyone to remember about closing all the producers all the time, thus
+        # this approach is better. Although it is still worth keeping in mind, that this will
+        # block GC from removing a no longer used producer unless closed properly but at least
+        # won't crash the VM upon closing the process
+        ObjectSpace.define_finalizer(id, proc { close })
+
+        @pid = Process.pid
+        @client = Builder.new.call(self, @config)
+        @status.connected!
+      end
+
+      @client
+    end
+
+    # Flushes the buffers in a sync way and closes the producer
+    def close
+      @closing_mutex.synchronize do
+        return unless @status.active?
+
+        @monitor.instrument(
+          'producer.closed',
+          producer: self
+        ) do
+          @status.closing!
+
+          # No need for auto-gc if everything got closed by us
+          # This should be used only in case a producer was not closed properly and forgotten
+          ObjectSpace.undefine_finalizer(id)
+
+          # Flush has it's own buffer mutex but even if it is blocked, flushing can still happen
+          # as we close the client after the flushing (even if blocked by the mutex)
+          flush(false)
+
+          # We should not close the client in several threads the same time
+          # It is safe to run it several times but not exactly the same moment
+          client.close
+
+          @status.closed!
+        end
+      end
+    end
+
+    # Ensures that we don't run any operations when the producer is not configured or when it
+    # was already closed
+    def ensure_active!
+      return if @status.active?
+
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closing? || @status.closed?
+
+      # This should never happen
+      raise Errors::StatusInvalidError, [id, @status.to_s]
+    end
+
+    # Ensures that the message we want to send out to Kafka is actually valid and that it can be
+    # sent there
+    # @param message [Hash] message we want to send
+    # @raise [Karafka::Errors::MessageInvalidError]
+    def validate_message!(message)
+      result = @contract.call(message)
+      return if result.success?
+
+      raise Errors::MessageInvalidError, [
+        result.errors.to_h,
+        message
+      ]
+    end
+  end
+end

data/lib/water_drop/producer/async.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for asynchronous producer operations
+    module Async
+      # Produces a message to Kafka and does not wait for results
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryHandle] delivery handle that might return the report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_async(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_async',
+          producer: self,
+          message: message
+        ) { client.produce(**message) }
+      end
+
+      # Produces many messages to Kafka and does not wait for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_async(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.produced_async',
+          producer: self,
+          messages: messages
+        ) do
+          messages.map { |message| client.produce(**message) }
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/buffer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for buffered operations
+    module Buffer
+      # Exceptions we catch when dispatching messages from a buffer
+      RESCUED_ERRORS = [
+        Rdkafka::RdkafkaError,
+        Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      ].freeze
+
+      private_constant :RESCUED_ERRORS
+
+      # Adds given message into the internal producer buffer without flushing it to Kafka
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def buffer(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.buffered',
+          producer: self,
+          message: message
+        ) { @messages << message }
+      end
+
+      # Adds given messages into the internal producer buffer without flushing them to Kafka
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def buffer_many(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.buffered',
+          producer: self,
+          messages: messages
+        ) do
+          messages.each { |message| @messages << message }
+          messages
+        end
+      end
+
+      # Flushes the internal buffer to Kafka in an async way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   flushed
+      def flush_async
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_async',
+          producer: self,
+          messages: @messages
+        ) { flush(false) }
+      end
+
+      # Flushes the internal buffer to Kafka in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
+      #   flushed
+      def flush_sync
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_sync',
+          producer: self,
+          messages: @messages
+        ) { flush(true) }
+      end
+
+      private
+
+      # Method for triggering the buffer
+      # @param sync [Boolean] should it flush in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle, Rdkafka::Producer::DeliveryReport>]
+      #   delivery handles for async or delivery reports for sync
+      # @raise [Errors::FlushFailureError] when there was a failure in flushing
+      # @note We use this method underneath to provide a different instrumentation for sync and
+      #   async flushing within the public API
+      def flush(sync)
+        data_for_dispatch = nil
+        dispatched = []
+
+        @buffer_mutex.synchronize do
+          data_for_dispatch = @messages
+          @messages = Concurrent::Array.new
+        end
+
+        dispatched = data_for_dispatch.map { |message| client.produce(**message) }
+
+        return dispatched unless sync
+
+        dispatched.map do |handler|
+          handler.wait(
+            max_wait_timeout: @config.max_wait_timeout,
+            wait_timeout: @config.wait_timeout
+          )
+        end
+      rescue *RESCUED_ERRORS => e
+        key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
+        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+
+        raise Errors::FlushFailureError.new(dispatched)
+      end
+    end
+  end
+end

data/lib/water_drop/producer/builder.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Class used to construct the rdkafka producer client
+    class Builder
+      # @param producer [Producer] not yet configured producer for which we want to
+      #   build the client
+      # @param config [Object] dry-configurable based configuration object
+      # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
+      #   when we don't want to dispatch any messages
+      def call(producer, config)
+        return DummyClient.new unless config.deliver
+
+        Rdkafka::Config.logger = config.logger
+        Rdkafka::Config.statistics_callback = build_statistics_callback(producer, config.monitor)
+
+        client = Rdkafka::Config.new(config.kafka.to_h).producer
+        client.delivery_callback = build_delivery_callback(producer, config.monitor)
+        client
+      end
+
+      private
+
+      # Creates a proc that we want to run upon each successful message delivery
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] delivery callback
+      def build_delivery_callback(producer, monitor)
+        lambda do |delivery_report|
+          monitor.instrument(
+            'message.acknowledged',
+            producer: producer,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+
+      # Creates a proc that we want to run upon each statistics callback execution
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] statistics callback
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      def build_statistics_callback(producer, monitor)
+        statistics_decorator = StatisticsDecorator.new
+
+        lambda do |statistics|
+          monitor.instrument(
+            'statistics.emitted',
+            producer: producer,
+            statistics: statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/dummy_client.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
+    # want to dispatch anything to Kafka
+    class DummyClient
+      # @return [DummyClient] dummy instance
+      def initialize
+        @counter = -1
+      end
+
+      # Dummy method for returning the delivery report
+      # @param _args [Object] anything that the delivery handle accepts
+      # @return [::Rdkafka::Producer::DeliveryReport]
+      def wait(*_args)
+        ::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      def respond_to_missing?(*_args)
+        true
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      # @return [self] returns self for chaining cases
+      def method_missing(*_args)
+        self || super
+      end
+    end
+  end
+end

data/lib/water_drop/producer/statistics_decorator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Many of the librdkafka statistics are absolute values instead of a gauge.
+    # This means, that for example number of messages sent is an absolute growing value
+    # instead of being a value of messages sent from the last statistics report.
+    # This decorator calculates the diff against previously emited stats, so we get also
+    # the diff together with the original values
+    class StatisticsDecorator
+      def initialize
+        @previous = {}.freeze
+      end
+
+      # @param emited_stats [Hash] original emited statistics
+      # @return [Hash] emited statistics extended with the diff data
+      # @note We modify the emited statistics, instead of creating new. Since we don't expose
+      #   any API to get raw data, users can just assume that the result of this decoration is the
+      #   proper raw stats that they can use
+      def call(emited_stats)
+        diff(
+          @previous,
+          emited_stats
+        )
+
+        @previous = emited_stats
+
+        emited_stats.freeze
+      end
+
+      private
+
+      # Calculates the diff of the provided values and modifies in place the emited statistics
+      #
+      # @param previous [Object] previous value from the given scope in which
+      #   we are
+      # @param current [Object] current scope from emitted statistics
+      # @return [Object] the diff if the values were numerics or the current scope
+      def diff(previous, current)
+        if current.is_a?(Hash)
+          # @note We cannot use #each_key as we modify the content of the current scope
+          #   in place (in case it's a hash)
+          current.keys.each do |key|
+            append(
+              current,
+              key,
+              diff((previous || {})[key], (current || {})[key])
+            )
+          end
+        end
+
+        if current.is_a?(Numeric) && previous.is_a?(Numeric)
+          current - previous
+        else
+          current
+        end
+      end
+
+      # Appends the result of the diff to a given key as long as the result is numeric
+      #
+      # @param current [Hash] current scope
+      # @param key [Symbol] key based on which we were diffing
+      # @param result [Object] diff result
+      def append(current, key, result)
+        return unless result.is_a?(Numeric)
+
+        current["#{key}_d"] = result
+      end
+    end
+  end
+end