waterdrop 1.4.0 → 2.0.1

data/lib/water_drop/producer/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Producer lifecycle status object representation
+    class Status
+      # States in which the producer can be
+      LIFECYCLE = %i[
+        initial
+        configured
+        connected
+        closing
+        closed
+      ].freeze
+
+      private_constant :LIFECYCLE
+
+      # Creates a new instance of status with the initial state
+      # @return [Status]
+      def initialize
+        @current = LIFECYCLE.first
+      end
+
+      # @return [Boolean] true if producer is in a active state. Active means, that we can start
+      #   sending messages. Actives states are connected (connection established) or configured,
+      #   which means, that producer is configured, but connection with Kafka is
+      #   not yet established.
+      def active?
+        connected? || configured?
+      end
+
+      # @return [String] current status as a string
+      def to_s
+        @current.to_s
+      end
+
+      LIFECYCLE.each do |state|
+        module_eval <<-RUBY, __FILE__, __LINE__ + 1
+          # @return [Boolean] true if current status is as we want, otherwise false
+          def #{state}?
+            @current == :#{state}
+          end
+
+          # Sets a given state as current
+          def #{state}!
+            @current = :#{state}
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/water_drop/producer/sync.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for synchronous producer operations
+    module Sync
+      # Produces a message to Kafka and waits for it to be delivered
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   handle is still pending
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_sync(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_sync',
+          producer: self,
+          message: message
+        ) do
+          client
+            .produce(**message)
+            .wait(
+              max_wait_timeout: @config.max_wait_timeout,
+              wait_timeout: @config.wait_timeout
+            )
+        end
+      end
+
+      # Produces many messages to Kafka and waits for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   some handles are still pending
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_sync(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+          messages
+            .map { |message| client.produce(**message) }
+            .map! do |handler|
+              handler.wait(
+                max_wait_timeout: @config.max_wait_timeout,
+                wait_timeout: @config.wait_timeout
+              )
+            end
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '1.4.0'
+  VERSION = '2.0.1'
 end

data/waterdrop.gemspec

@@ -14,16 +14,16 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/karafka/waterdrop'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
-  spec.license       = 'MIT'
+  spec.license       = 'LGPL-3.0'
 
-  spec.add_dependency 'delivery_boy', '>= 0.2', '< 2.x'
+  spec.add_dependency 'concurrent-ruby', '>= 1.1'
   spec.add_dependency 'dry-configurable', '~> 0.8'
   spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'dry-validation', '~> 1.3'
+  spec.add_dependency 'rdkafka', '>= 0.6.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
-  spec.required_ruby_version = '>= 2.5.0'
+  spec.required_ruby_version = '>= 2.6.0'
 
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 2.0.1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -34,28 +34,22 @@ cert_chain:
   2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
   nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2020-08-25 00:00:00.000000000 Z
+date: 2021-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: delivery_boy
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.x
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.x
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -90,28 +84,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -134,38 +128,45 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
+- ".diffend.yml"
 - ".github/FUNDING.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
-- MIT-LICENCE
+- LICENSE
 - README.md
 - certs/mensfeld.pem
 - config/errors.yml
+- docker-compose.yml
 - lib/water_drop.rb
-- lib/water_drop/async_producer.rb
-- lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
-- lib/water_drop/config_applier.rb
 - lib/water_drop/contracts.rb
 - lib/water_drop/contracts/config.rb
-- lib/water_drop/contracts/message_options.rb
+- lib/water_drop/contracts/message.rb
 - lib/water_drop/errors.rb
+- lib/water_drop/instrumentation.rb
 - lib/water_drop/instrumentation/monitor.rb
 - lib/water_drop/instrumentation/stdout_listener.rb
-- lib/water_drop/sync_producer.rb
+- lib/water_drop/producer.rb
+- lib/water_drop/producer/async.rb
+- lib/water_drop/producer/buffer.rb
+- lib/water_drop/producer/builder.rb
+- lib/water_drop/producer/dummy_client.rb
+- lib/water_drop/producer/statistics_decorator.rb
+- lib/water_drop/producer/status.rb
+- lib/water_drop/producer/sync.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
 - log/.gitkeep
 - waterdrop.gemspec
 homepage: https://github.com/karafka/waterdrop
 licenses:
-- MIT
+- LGPL-3.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -175,14 +176,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.2.15
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/.travis.yml

@@ -1,35 +0,0 @@
-services:
-  - docker
-
-dist: trusty
-cache: bundler
-
-git:
-  depth: false
-
-test: &test
-  stage: Test
-  language: ruby
-  before_install:
-    - yes | gem update --system
-  script: bundle exec rspec
-
-jobs:
-  include:
-    - <<: *test
-      rvm: 2.7.1
-    - <<: *test
-      rvm: 2.6.6
-    - <<: *test
-      rvm: 2.5.8
-
-    - stage: coditsu
-      language: ruby
-      rvm: 2.7.1
-      before_install:
-        - yes | gem update --system
-      script: \curl -sSL https://api.coditsu.io/run/ci | bash
-
-stages:
-  - test
-  - coditsu

data/MIT-LICENCE

@@ -1,18 +0,0 @@
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/water_drop/async_producer.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-# WaterDrop library
-module WaterDrop
-  # Async producer for messages
-  class AsyncProducer < BaseProducer
-    # Performs message delivery using deliver_async method
-    # @param message [String] message that we want to send to Kafka
-    # @param options [Hash] options (including topic) for producer
-    # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
-    #   somehow invalid and we cannot perform delivery because of that
-    def self.call(message, options)
-      attempts_count ||= 0
-      attempts_count += 1
-
-      validate!(options)
-      return unless WaterDrop.config.deliver
-
-      d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
-
-      DeliveryBoy.send(d_method, message, **options)
-    rescue Kafka::Error => e
-      graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
-    end
-  end
-end

data/lib/water_drop/base_producer.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Base messages producer that contains all the logic that is exactly the same for both
-  # sync and async producers
-  class BaseProducer
-    # Contract for checking the correctness of the provided data that someone wants to
-    # dispatch to Kafka
-    SCHEMA = Contracts::MessageOptions.new.freeze
-
-    private_constant :SCHEMA
-
-    class << self
-      private
-
-      # Runs the message options validations and raises an error if anything is invalid
-      # @param options [Hash] hash that we want to validate
-      # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
-      #   somehow invalid and we cannot perform delivery because of that
-      def validate!(options)
-        validation_result = SCHEMA.call(options)
-        return true if validation_result.success?
-
-        raise Errors::InvalidMessageOptions, validation_result.errors
-      end
-
-      # Upon failed delivery, we may try to resend a message depending on the attempt number
-      #   or re-raise an error if we're unable to do that after given number of retries
-      #   This method checks that and also instruments errors and retries for the delivery
-      # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
-      # @param message [String] message that we want to send to Kafka
-      # @param options [Hash] options (including topic) for producer
-      # @param error [Kafka::Error] error that occurred
-      # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
-      #   was the final one and we should deal with the fact, that we cannot deliver a given
-      #   message
-      def graceful_attempt?(attempts_count, message, options, error)
-        scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
-        payload = {
-          caller: self,
-          message: message,
-          options: options,
-          error: error,
-          attempts_count: attempts_count
-        }
-
-        if attempts_count > WaterDrop.config.kafka.max_retries
-          WaterDrop.monitor.instrument("#{scope}.error", payload)
-          false
-        else
-          WaterDrop.monitor.instrument("#{scope}.retry", payload)
-          true
-        end
-      end
-    end
-  end
-end

data/lib/water_drop/config_applier.rb

@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Engine used to propagate config application to DeliveryBoy with corner case handling
-  module ConfigApplier
-    class << self
-      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
-      # @param settings [Hash] hash with WaterDrop settings
-      def call(delivery_boy_config, settings)
-        # Recursive lambda for mapping config down to delivery boy
-        settings.each do |key, value|
-          call(delivery_boy_config, value) && next if value.is_a?(Hash)
-
-          # If this is a special case that needs manual setup instead of a direct reassignment
-          if respond_to?(key, true)
-            send(key, delivery_boy_config, value)
-          else
-            # If this setting is our internal one, we should not sync it with the delivery boy
-            next unless delivery_boy_config.respond_to?(:"#{key}=")
-
-            delivery_boy_config.public_send(:"#{key}=", value)
-          end
-        end
-      end
-
-      private
-
-      # Extra setup for the compression codec as it behaves differently than other settings
-      # that are ported 1:1 from ruby-kafka
-      # For some crazy reason, delivery boy requires compression codec as a string, when
-      # ruby-kafka as a symbol. We follow ruby-kafka internal design, so we had to mimic
-      # that by assigning a string version that down the road will be symbolized again
-      # by delivery boy
-      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
-      # @param codec_name [Symbol] codec name as a symbol
-      def compression_codec(delivery_boy_config, codec_name)
-        # If there is no compression codec, we don't apply anything
-        return unless codec_name
-
-        delivery_boy_config.compression_codec = codec_name.to_s
-      end
-
-      # We use the "seed_brokers" name and DeliveryBoy uses "brokers" so we pass the values
-      #   manually
-      # @param delivery_boy_config [DeliveryBoy::Config] delivery boy config instance
-      # @param seed_brokers [Array<String>] kafka seed brokers
-      def seed_brokers(delivery_boy_config, seed_brokers)
-        delivery_boy_config.brokers = seed_brokers
-      end
-    end
-  end
-end