waterdrop 1.4.2 → 2.0.2

data/lib/water_drop/producer/async.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for asynchronous producer operations
+    module Async
+      # Produces a message to Kafka and does not wait for results
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryHandle] delivery handle that might return the report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_async(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_async',
+          producer: self,
+          message: message
+        ) { client.produce(**message) }
+      end
+
+      # Produces many messages to Kafka and does not wait for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] deliveries handles
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_async(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.produced_async',
+          producer: self,
+          messages: messages
+        ) do
+          messages.map { |message| client.produce(**message) }
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/buffer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for buffered operations
+    module Buffer
+      # Exceptions we catch when dispatching messages from a buffer
+      RESCUED_ERRORS = [
+        Rdkafka::RdkafkaError,
+        Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      ].freeze
+
+      private_constant :RESCUED_ERRORS
+
+      # Adds given message into the internal producer buffer without flushing it to Kafka
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def buffer(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.buffered',
+          producer: self,
+          message: message
+        ) { @messages << message }
+      end
+
+      # Adds given messages into the internal producer buffer without flushing them to Kafka
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def buffer_many(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.buffered',
+          producer: self,
+          messages: messages
+        ) do
+          messages.each { |message| @messages << message }
+          messages
+        end
+      end
+
+      # Flushes the internal buffer to Kafka in an async way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   flushed
+      def flush_async
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_async',
+          producer: self,
+          messages: @messages
+        ) { flush(false) }
+      end
+
+      # Flushes the internal buffer to Kafka in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
+      #   flushed
+      def flush_sync
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_sync',
+          producer: self,
+          messages: @messages
+        ) { flush(true) }
+      end
+
+      private
+
+      # Method for triggering the buffer
+      # @param sync [Boolean] should it flush in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle, Rdkafka::Producer::DeliveryReport>]
+      #   delivery handles for async or delivery reports for sync
+      # @raise [Errors::FlushFailureError] when there was a failure in flushing
+      # @note We use this method underneath to provide a different instrumentation for sync and
+      #   async flushing within the public API
+      def flush(sync)
+        data_for_dispatch = nil
+        dispatched = []
+
+        @buffer_mutex.synchronize do
+          data_for_dispatch = @messages
+          @messages = Concurrent::Array.new
+        end
+
+        dispatched = data_for_dispatch.map { |message| client.produce(**message) }
+
+        return dispatched unless sync
+
+        dispatched.map do |handler|
+          handler.wait(
+            max_wait_timeout: @config.max_wait_timeout,
+            wait_timeout: @config.wait_timeout
+          )
+        end
+      rescue *RESCUED_ERRORS => e
+        key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
+        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+
+        raise Errors::FlushFailureError.new(dispatched)
+      end
+    end
+  end
+end

data/lib/water_drop/producer/builder.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Class used to construct the rdkafka producer client
+    class Builder
+      # @param producer [Producer] not yet configured producer for which we want to
+      #   build the client
+      # @param config [Object] dry-configurable based configuration object
+      # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
+      #   when we don't want to dispatch any messages
+      def call(producer, config)
+        return DummyClient.new unless config.deliver
+
+        Rdkafka::Config.logger = config.logger
+        Rdkafka::Config.statistics_callback = build_statistics_callback(producer, config.monitor)
+
+        client = Rdkafka::Config.new(config.kafka.to_h).producer
+        client.delivery_callback = build_delivery_callback(producer, config.monitor)
+        client
+      end
+
+      private
+
+      # Creates a proc that we want to run upon each successful message delivery
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] delivery callback
+      def build_delivery_callback(producer, monitor)
+        lambda do |delivery_report|
+          monitor.instrument(
+            'message.acknowledged',
+            producer: producer,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+
+      # Creates a proc that we want to run upon each statistics callback execution
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] statistics callback
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      def build_statistics_callback(producer, monitor)
+        statistics_decorator = StatisticsDecorator.new
+
+        lambda do |statistics|
+          monitor.instrument(
+            'statistics.emitted',
+            producer: producer,
+            statistics: statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/dummy_client.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
+    # want to dispatch anything to Kafka
+    class DummyClient
+      # @return [DummyClient] dummy instance
+      def initialize
+        @counter = -1
+      end
+
+      # Dummy method for returning the delivery report
+      # @param _args [Object] anything that the delivery handle accepts
+      # @return [::Rdkafka::Producer::DeliveryReport]
+      def wait(*_args)
+        ::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      def respond_to_missing?(*_args)
+        true
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      # @return [self] returns self for chaining cases
+      def method_missing(*_args)
+        self || super
+      end
+    end
+  end
+end

data/lib/water_drop/producer/statistics_decorator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Many of the librdkafka statistics are absolute values instead of a gauge.
+    # This means, that for example number of messages sent is an absolute growing value
+    # instead of being a value of messages sent from the last statistics report.
+    # This decorator calculates the diff against previously emited stats, so we get also
+    # the diff together with the original values
+    class StatisticsDecorator
+      def initialize
+        @previous = {}.freeze
+      end
+
+      # @param emited_stats [Hash] original emited statistics
+      # @return [Hash] emited statistics extended with the diff data
+      # @note We modify the emited statistics, instead of creating new. Since we don't expose
+      #   any API to get raw data, users can just assume that the result of this decoration is the
+      #   proper raw stats that they can use
+      def call(emited_stats)
+        diff(
+          @previous,
+          emited_stats
+        )
+
+        @previous = emited_stats
+
+        emited_stats.freeze
+      end
+
+      private
+
+      # Calculates the diff of the provided values and modifies in place the emited statistics
+      #
+      # @param previous [Object] previous value from the given scope in which
+      #   we are
+      # @param current [Object] current scope from emitted statistics
+      # @return [Object] the diff if the values were numerics or the current scope
+      def diff(previous, current)
+        if current.is_a?(Hash)
+          # @note We cannot use #each_key as we modify the content of the current scope
+          #   in place (in case it's a hash)
+          current.keys.each do |key|
+            append(
+              current,
+              key,
+              diff((previous || {})[key], (current || {})[key])
+            )
+          end
+        end
+
+        if current.is_a?(Numeric) && previous.is_a?(Numeric)
+          current - previous
+        else
+          current
+        end
+      end
+
+      # Appends the result of the diff to a given key as long as the result is numeric
+      #
+      # @param current [Hash] current scope
+      # @param key [Symbol] key based on which we were diffing
+      # @param result [Object] diff result
+      def append(current, key, result)
+        return unless result.is_a?(Numeric)
+
+        current["#{key}_d"] = result
+      end
+    end
+  end
+end

data/lib/water_drop/producer/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Producer lifecycle status object representation
+    class Status
+      # States in which the producer can be
+      LIFECYCLE = %i[
+        initial
+        configured
+        connected
+        closing
+        closed
+      ].freeze
+
+      private_constant :LIFECYCLE
+
+      # Creates a new instance of status with the initial state
+      # @return [Status]
+      def initialize
+        @current = LIFECYCLE.first
+      end
+
+      # @return [Boolean] true if producer is in a active state. Active means, that we can start
+      #   sending messages. Actives states are connected (connection established) or configured,
+      #   which means, that producer is configured, but connection with Kafka is
+      #   not yet established.
+      def active?
+        connected? || configured?
+      end
+
+      # @return [String] current status as a string
+      def to_s
+        @current.to_s
+      end
+
+      LIFECYCLE.each do |state|
+        module_eval <<-RUBY, __FILE__, __LINE__ + 1
+          # @return [Boolean] true if current status is as we want, otherwise false
+          def #{state}?
+            @current == :#{state}
+          end
+
+          # Sets a given state as current
+          def #{state}!
+            @current = :#{state}
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/water_drop/producer/sync.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for synchronous producer operations
+    module Sync
+      # Produces a message to Kafka and waits for it to be delivered
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   handle is still pending
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_sync(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_sync',
+          producer: self,
+          message: message
+        ) do
+          client
+            .produce(**message)
+            .wait(
+              max_wait_timeout: @config.max_wait_timeout,
+              wait_timeout: @config.wait_timeout
+            )
+        end
+      end
+
+      # Produces many messages to Kafka and waits for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   some handles are still pending
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_sync(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+          messages
+            .map { |message| client.produce(**message) }
+            .map! do |handler|
+              handler.wait(
+                max_wait_timeout: @config.max_wait_timeout,
+                wait_timeout: @config.wait_timeout
+              )
+            end
+        end
+      end
+    end
+  end
+end