waterdrop 1.4.0 → 2.0.0.rc1

data/lib/water_drop/producer/buffer.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for buffered operations
+    module Buffer
+      # Exceptions we catch when dispatching messages from a buffer
+      RESCUED_ERRORS = [
+        Rdkafka::RdkafkaError,
+        Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+      ].freeze
+
+      private_constant :RESCUED_ERRORS
+
+      # Adds given message into the internal producer buffer without flushing it to Kafka
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def buffer(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.buffered',
+          producer: self,
+          message: message
+        ) { @messages << message }
+      end
+
+      # Adds given messages into the internal producer buffer without flushing them to Kafka
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def buffer_many(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument(
+          'messages.buffered',
+          producer: self,
+          messages: messages
+        ) do
+          messages.each { |message| @messages << message }
+          messages
+        end
+      end
+
+      # Flushes the internal buffer to Kafka in an async way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle>] delivery handles for messages that were
+      #   flushed
+      def flush_async
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_async',
+          producer: self,
+          messages: @messages
+        ) { flush(false) }
+      end
+
+      # Flushes the internal buffer to Kafka in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports for messages that were
+      #   flushed
+      def flush_sync
+        ensure_active!
+
+        @monitor.instrument(
+          'buffer.flushed_sync',
+          producer: self,
+          messages: @messages
+        ) { flush(true) }
+      end
+
+      private
+
+      # Method for triggering the buffer
+      # @param sync [Boolean] should it flush in a sync way
+      # @return [Array<Rdkafka::Producer::DeliveryHandle, Rdkafka::Producer::DeliveryReport>]
+      #   delivery handles for async or delivery reports for sync
+      # @raise [Errors::FlushFailureError] when there was a failure in flushing
+      # @note We use this method underneath to provide a different instrumentation for sync and
+      #   async flushing within the public API
+      def flush(sync)
+        data_for_dispatch = nil
+        dispatched = []
+
+        @buffer_mutex.synchronize do
+          data_for_dispatch = @messages
+          @messages = Concurrent::Array.new
+        end
+
+        dispatched = data_for_dispatch.map { |message| client.produce(**message) }
+
+        return dispatched unless sync
+
+        dispatched.map do |handler|
+          handler.wait(
+            max_wait_timeout: @config.max_wait_timeout,
+            wait_timeout: @config.wait_timeout
+          )
+        end
+      rescue *RESCUED_ERRORS => e
+        key = sync ? 'buffer.flushed_sync.error' : 'buffer.flush_async.error'
+        @monitor.instrument(key, producer: self, error: e, dispatched: dispatched)
+
+        raise Errors::FlushFailureError.new(dispatched)
+      end
+    end
+  end
+end

data/lib/water_drop/producer/builder.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Class used to construct the rdkafka producer client
+    class Builder
+      # @param producer [Producer] not yet configured producer for which we want to
+      #   build the client
+      # @param config [Object] dry-configurable based configuration object
+      # @return [Rdkafka::Producer, Producer::DummyClient] raw rdkafka producer or a dummy producer
+      #   when we don't want to dispatch any messages
+      def call(producer, config)
+        return DummyClient.new unless config.deliver
+
+        Rdkafka::Config.logger = config.logger
+        Rdkafka::Config.statistics_callback = build_statistics_callback(producer, config.monitor)
+
+        client = Rdkafka::Config.new(config.kafka.to_h).producer
+        client.delivery_callback = build_delivery_callback(producer, config.monitor)
+        client
+      end
+
+      private
+
+      # Creates a proc that we want to run upon each successful message delivery
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] delivery callback
+      def build_delivery_callback(producer, monitor)
+        lambda do |delivery_report|
+          monitor.instrument(
+            'message.acknowledged',
+            producer: producer,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition
+          )
+        end
+      end
+
+      # Creates a proc that we want to run upon each statistics callback execution
+      #
+      # @param producer [Producer]
+      # @param monitor [Object] monitor we want to use
+      # @return [Proc] statistics callback
+      # @note We decorate the statistics with our own decorator because some of the metrics from
+      #   rdkafka are absolute. For example number of sent messages increases not in reference to
+      #   previous statistics emit but from the beginning of the process. We decorate it with diff
+      #   of all the numeric values against the data from the previous callback emit
+      def build_statistics_callback(producer, monitor)
+        statistics_decorator = StatisticsDecorator.new
+
+        lambda do |statistics|
+          monitor.instrument(
+            'statistics.emitted',
+            producer: producer,
+            statistics: statistics_decorator.call(statistics)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/producer/dummy_client.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # A dummy client that is supposed to be used instead of Rdkafka::Producer in case we don't
+    # want to dispatch anything to Kafka
+    class DummyClient
+      # @return [DummyClient] dummy instance
+      def initialize
+        @counter = -1
+      end
+
+      # Dummy method for returning the delivery report
+      # @param _args [Object] anything that the delivery handle accepts
+      # @return [::Rdkafka::Producer::DeliveryReport]
+      def wait(*_args)
+        ::Rdkafka::Producer::DeliveryReport.new(0, @counter += 1)
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      def respond_to_missing?(*_args)
+        true
+      end
+
+      # @param _args [Object] anything really, this dummy is suppose to support anything
+      # @return [self] returns self for chaining cases
+      def method_missing(*_args)
+        self || super
+      end
+    end
+  end
+end

data/lib/water_drop/producer/statistics_decorator.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Many of the librdkafka statistics are absolute values instead of a gauge.
+    # This means, that for example number of messages sent is an absolute growing value
+    # instead of being a value of messages sent from the last statistics report.
+    # This decorator calculates the diff against previously emited stats, so we get also
+    # the diff together with the original values
+    class StatisticsDecorator
+      def initialize
+        @previous = {}.freeze
+      end
+
+      # @param emited_stats [Hash] original emited statistics
+      # @return [Hash] emited statistics extended with the diff data
+      # @note We modify the emited statistics, instead of creating new. Since we don't expose
+      #   any API to get raw data, users can just assume that the result of this decoration is the
+      #   proper raw stats that they can use
+      def call(emited_stats)
+        diff(
+          @previous,
+          emited_stats
+        )
+
+        @previous = emited_stats
+
+        emited_stats.freeze
+      end
+
+      private
+
+      # Calculates the diff of the provided values and modifies in place the emited statistics
+      #
+      # @param previous [Object] previous value from the given scope in which
+      #   we are
+      # @param current [Object] current scope from emitted statistics
+      # @return [Object] the diff if the values were numerics or the current scope
+      def diff(previous, current)
+        if current.is_a?(Hash)
+          # @note We cannot use #each_key as we modify the content of the current scope
+          #   in place (in case it's a hash)
+          current.keys.each do |key|
+            append(
+              current,
+              key,
+              diff((previous || {})[key], (current || {})[key])
+            )
+          end
+        end
+
+        if current.is_a?(Numeric) && previous.is_a?(Numeric)
+          current - previous
+        else
+          current
+        end
+      end
+
+      # Appends the result of the diff to a given key as long as the result is numeric
+      #
+      # @param current [Hash] current scope
+      # @param key [Symbol] key based on which we were diffing
+      # @param result [Object] diff result
+      def append(current, key, result)
+        return unless result.is_a?(Numeric)
+
+        current["#{key}_d"] = result
+      end
+    end
+  end
+end

data/lib/water_drop/producer/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Producer lifecycle status object representation
+    class Status
+      # States in which the producer can be
+      LIFECYCLE = %i[
+        initial
+        configured
+        connected
+        closing
+        closed
+      ].freeze
+
+      private_constant :LIFECYCLE
+
+      # Creates a new instance of status with the initial state
+      # @return [Status]
+      def initialize
+        @current = LIFECYCLE.first
+      end
+
+      # @return [Boolean] true if producer is in a active state. Active means, that we can start
+      #   sending messages. Actives states are connected (connection established) or configured,
+      #   which means, that producer is configured, but connection with Kafka is
+      #   not yet established.
+      def active?
+        connected? || configured?
+      end
+
+      # @return [String] current status as a string
+      def to_s
+        @current.to_s
+      end
+
+      LIFECYCLE.each do |state|
+        module_eval <<-RUBY, __FILE__, __LINE__ + 1
+          # @return [Boolean] true if current status is as we want, otherwise false
+          def #{state}?
+            @current == :#{state}
+          end
+
+          # Sets a given state as current
+          def #{state}!
+            @current = :#{state}
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/water_drop/producer/sync.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for synchronous producer operations
+    module Sync
+      # Produces a message to Kafka and waits for it to be delivered
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   handle is still pending
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_sync(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_sync',
+          producer: self,
+          message: message
+        ) do
+          client
+            .produce(**message)
+            .wait(
+              max_wait_timeout: @config.max_wait_timeout,
+              wait_timeout: @config.wait_timeout
+            )
+        end
+      end
+
+      # Produces many messages to Kafka and waits for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   some handles are still pending
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_sync(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+          messages
+            .map { |message| client.produce(**message) }
+            .map! do |handler|
+              handler.wait(
+                max_wait_timeout: @config.max_wait_timeout,
+                wait_timeout: @config.wait_timeout
+              )
+            end
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '1.4.0'
+  VERSION = '2.0.0.rc1'
 end

data/waterdrop.gemspec

@@ -14,13 +14,13 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/karafka/waterdrop'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
-  spec.license       = 'MIT'
+  spec.license       = 'LGPL-3.0'
 
-  spec.add_dependency 'delivery_boy', '>= 0.2', '< 2.x'
+  spec.add_dependency 'concurrent-ruby', '>= 1.1'
   spec.add_dependency 'dry-configurable', '~> 0.8'
   spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'dry-validation', '~> 1.3'
+  spec.add_dependency 'rdkafka', '>= 0.6.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
   spec.required_ruby_version = '>= 2.5.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,28 +34,22 @@ cert_chain:
   2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
   nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2020-08-25 00:00:00.000000000 Z
+date: 2020-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: delivery_boy
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.x
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.x
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -90,28 +84,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -134,38 +128,45 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
+- ".diffend.yml"
 - ".github/FUNDING.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
-- MIT-LICENCE
+- LICENSE
 - README.md
 - certs/mensfeld.pem
 - config/errors.yml
+- docker-compose.yml
 - lib/water_drop.rb
-- lib/water_drop/async_producer.rb
-- lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
-- lib/water_drop/config_applier.rb
 - lib/water_drop/contracts.rb
 - lib/water_drop/contracts/config.rb
-- lib/water_drop/contracts/message_options.rb
+- lib/water_drop/contracts/message.rb
 - lib/water_drop/errors.rb
+- lib/water_drop/instrumentation.rb
 - lib/water_drop/instrumentation/monitor.rb
 - lib/water_drop/instrumentation/stdout_listener.rb
-- lib/water_drop/sync_producer.rb
+- lib/water_drop/producer.rb
+- lib/water_drop/producer/async.rb
+- lib/water_drop/producer/buffer.rb
+- lib/water_drop/producer/builder.rb
+- lib/water_drop/producer/dummy_client.rb
+- lib/water_drop/producer/statistics_decorator.rb
+- lib/water_drop/producer/status.rb
+- lib/water_drop/producer/sync.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
 - log/.gitkeep
 - waterdrop.gemspec
 homepage: https://github.com/karafka/waterdrop
 licenses:
-- MIT
+- LGPL-3.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -178,9 +179,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubygems_version: 3.1.4
 signing_key: