waterdrop 1.3.1 → 2.0.0.rc1

data/lib/water_drop/producer/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Producer lifecycle status object representation
+    class Status
+      # States in which the producer can be
+      LIFECYCLE = %i[
+        initial
+        configured
+        connected
+        closing
+        closed
+      ].freeze
+
+      private_constant :LIFECYCLE
+
+      # Creates a new instance of status with the initial state
+      # @return [Status]
+      def initialize
+        @current = LIFECYCLE.first
+      end
+
+      # @return [Boolean] true if producer is in a active state. Active means, that we can start
+      #   sending messages. Actives states are connected (connection established) or configured,
+      #   which means, that producer is configured, but connection with Kafka is
+      #   not yet established.
+      def active?
+        connected? || configured?
+      end
+
+      # @return [String] current status as a string
+      def to_s
+        @current.to_s
+      end
+
+      LIFECYCLE.each do |state|
+        module_eval <<-RUBY, __FILE__, __LINE__ + 1
+          # @return [Boolean] true if current status is as we want, otherwise false
+          def #{state}?
+            @current == :#{state}
+          end
+
+          # Sets a given state as current
+          def #{state}!
+            @current = :#{state}
+          end
+        RUBY
+      end
+    end
+  end
+end

data/lib/water_drop/producer/sync.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Component for synchronous producer operations
+    module Sync
+      # Produces a message to Kafka and waits for it to be delivered
+      #
+      # @param message [Hash] hash that complies with the {Contracts::Message} contract
+      #
+      # @return [Rdkafka::Producer::DeliveryReport] delivery report
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the message to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   handle is still pending
+      # @raise [Errors::MessageInvalidError] When provided message details are invalid and the
+      #   message could not be sent to Kafka
+      def produce_sync(message)
+        ensure_active!
+        validate_message!(message)
+
+        @monitor.instrument(
+          'message.produced_sync',
+          producer: self,
+          message: message
+        ) do
+          client
+            .produce(**message)
+            .wait(
+              max_wait_timeout: @config.max_wait_timeout,
+              wait_timeout: @config.wait_timeout
+            )
+        end
+      end
+
+      # Produces many messages to Kafka and waits for them to be delivered
+      #
+      # @param messages [Array<Hash>] array with messages that comply with the
+      #   {Contracts::Message} contract
+      #
+      # @return [Array<Rdkafka::Producer::DeliveryReport>] delivery reports
+      #
+      # @raise [Rdkafka::RdkafkaError] When adding the messages to rdkafka's queue failed
+      # @raise [Rdkafka::Producer::WaitTimeoutError] When the timeout has been reached and the
+      #   some handles are still pending
+      # @raise [Errors::MessageInvalidError] When any of the provided messages details are invalid
+      #   and the message could not be sent to Kafka
+      def produce_many_sync(messages)
+        ensure_active!
+        messages.each { |message| validate_message!(message) }
+
+        @monitor.instrument('messages.produced_sync', producer: self, messages: messages) do
+          messages
+            .map { |message| client.produce(**message) }
+            .map! do |handler|
+              handler.wait(
+                max_wait_timeout: @config.max_wait_timeout,
+                wait_timeout: @config.wait_timeout
+              )
+            end
+        end
+      end
+    end
+  end
+end

data/lib/water_drop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '1.3.1'
+  VERSION = '2.0.0.rc1'
 end

data/waterdrop.gemspec

@@ -14,13 +14,13 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/karafka/waterdrop'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
-  spec.license       = 'MIT'
+  spec.license       = 'LGPL-3.0'
 
-  spec.add_dependency 'delivery_boy', '~> 0.2'
+  spec.add_dependency 'concurrent-ruby', '>= 1.1'
   spec.add_dependency 'dry-configurable', '~> 0.8'
   spec.add_dependency 'dry-monitor', '~> 0.3'
-  spec.add_dependency 'dry-validation', '~> 1.2'
-  spec.add_dependency 'ruby-kafka', '>= 0.7.8'
+  spec.add_dependency 'dry-validation', '~> 1.3'
+  spec.add_dependency 'rdkafka', '>= 0.6.0'
   spec.add_dependency 'zeitwerk', '~> 2.1'
 
   spec.required_ruby_version = '>= 2.5.0'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 2.0.0.rc1
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -11,45 +11,45 @@ cert_chain:
 - |
   -----BEGIN CERTIFICATE-----
   MIIEODCCAqCgAwIBAgIBATANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDDBhtYWNp
-  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMTkwNzMwMTQ1NDU0WhcNMjAwNzI5MTQ1
-  NDU0WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
-  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQC9fCwtaHZG2SyyNXiH8r0QbJQx/xxl
-  dkvwWz9QGJO+O8rEx20FB1Ab+MVkfOscwIv5jWpmk1U9whzDPl1uFtIbgu+sk+Zb
-  uQlZyK/DPN6c+/BbBL+RryTBRyvkPLoCVwm7uxc/JZ1n4AI6eF4cCZ2ieZ9QgQbU
-  MQs2QPqs9hT50Ez/40GnOdadVfiDDGz+NME2C4ms0BriXwZ1tcRTfJIHe2xjIbbb
-  y5qRGfsLKcgMzvLQR24olixyX1MR0s4+Wveq3QL/gBhL4veUcv+UABJA8IJR0kyB
-  seHHutusiwZ1v3SjjjW1xLLrc2ARV0mgCb0WaK2T4iA3oFTGLh6Ydz8LNl31KQFv
-  94nRd8IhmJxrhQ6dQ/WT9IXoa5S9lfT5lPJeINemH4/6QPABzf9W2IZlCdI9wCdB
-  TBaw57MKneGAYZiKjw6OALSy2ltQUCl3RqFl3VP7n8uFy1U987Q5VIIQ3O1UUsQD
-  Oe/h+r7GUU4RSPKgPlrwvW9bD/UQ+zF51v8CAwEAAaN3MHUwCQYDVR0TBAIwADAL
-  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFJNIBHdfEUD7TqHqIer2YhWaWhwcMB0GA1Ud
+  ZWovREM9bWVuc2ZlbGQvREM9cGwwHhcNMjAwODExMDkxNTM3WhcNMjEwODExMDkx
+  NTM3WjAjMSEwHwYDVQQDDBhtYWNpZWovREM9bWVuc2ZlbGQvREM9cGwwggGiMA0G
+  CSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQDCpXsCgmINb6lHBXXBdyrgsBPSxC4/
+  2H+weJ6L9CruTiv2+2/ZkQGtnLcDgrD14rdLIHK7t0o3EKYlDT5GhD/XUVhI15JE
+  N7IqnPUgexe1fbZArwQ51afxz2AmPQN2BkB2oeQHXxnSWUGMhvcEZpfbxCCJH26w
+  hS0Ccsma8yxA6hSlGVhFVDuCr7c2L1di6cK2CtIDpfDaWqnVNJEwBYHIxrCoWK5g
+  sIGekVt/admS9gRhIMaIBg+Mshth5/DEyWO2QjteTodItlxfTctrfmiAl8X8T5JP
+  VXeLp5SSOJ5JXE80nShMJp3RFnGw5fqjX/ffjtISYh78/By4xF3a25HdWH9+qO2Z
+  tx0wSGc9/4gqNM0APQnjN/4YXrGZ4IeSjtE+OrrX07l0TiyikzSLFOkZCAp8oBJi
+  Fhlosz8xQDJf7mhNxOaZziqASzp/hJTU/tuDKl5+ql2icnMv5iV/i6SlmvU29QNg
+  LCV71pUv0pWzN+OZbHZKWepGhEQ3cG9MwvkCAwEAAaN3MHUwCQYDVR0TBAIwADAL
+  BgNVHQ8EBAMCBLAwHQYDVR0OBBYEFImGed2AXS070ohfRidiCEhXEUN+MB0GA1Ud
   EQQWMBSBEm1hY2llakBtZW5zZmVsZC5wbDAdBgNVHRIEFjAUgRJtYWNpZWpAbWVu
-  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKA4eqko6BTNhlysip6rfBkVTGri
-  ZXsL+kRb2hLvsQJS/kLyM21oMlu+LN0aPj3qEFR8mE/YeDD8rLAfruBRTltPNbR7
-  xA5eE1gkxY5LfExUtK3b2wPqfmo7mZgfcsMwfYg/tUXw1WpBCnrhAJodpGH6SXmp
-  A40qFUZst0vjiOoO+aTblIHPmMJXoZ3K42dTlNKlEiDKUWMRKSgpjjYGEYalFNWI
-  hHfCz2r8L2t+dYdMZg1JGbEkq4ADGsAA8ioZIpJd7V4hI17u5TCdi7X5wh/0gN0E
-  CgP+nLox3D+l2q0QuQEkayr+auFYkzTCkF+BmEk1D0Ru4mcf3F4CJvEmW4Pzbjqt
-  i1tsCWPtJ4E/UUKnKaWKqGbjrjHJ0MuShYzHkodox5IOiCXIQg+1+YSzfXUV6WEK
-  KJG/fhg1JV5vVDdVy6x+tv5SQ5ctU0feCsVfESi3rE3zRd+nvzE9HcZ5aXeL1UtJ
-  nT5Xrioegu2w1jPyVEgyZgTZC5rvD0nNS5sFNQ==
+  c2ZlbGQucGwwDQYJKoZIhvcNAQELBQADggGBAKiHpwoENVrMi94V1zD4o8/6G3AU
+  gWz4udkPYHTZLUy3dLznc/sNjdkJFWT3E6NKYq7c60EpJ0m0vAEg5+F5pmNOsvD3
+  2pXLj9kisEeYhR516HwXAvtngboUcb75skqvBCU++4Pu7BRAPjO1/ihLSBexbwSS
+  fF+J5OWNuyHHCQp+kGPLtXJe2yUYyvSWDj3I2//Vk0VhNOIlaCS1+5/P3ZJThOtm
+  zJUBI7h3HgovwRpcnmk2mXTmU4Zx/bCzX8EA6VY0khEvnmiq7S6eBF0H9qH8KyQ6
+  EkVLpvmUDFcf/uNaBQdazEMB5jYtwoA8gQlANETNGPi51KlkukhKgaIEDMkBDJOx
+  65N7DzmkcyY0/GwjIVIxmRhcrCt1YeCUElmfFx0iida1/YRm6sB2AXqScc1+ECRi
+  2DND//YJUikn1zwbz1kT70XmHd97B4Eytpln7K+M1u2g1pHVEPW4owD/ammXNpUy
+  nt70FcDD4yxJQ+0YNiHd0N8IcVBM1TMIVctMNQ==
   -----END CERTIFICATE-----
-date: 2019-10-21 00:00:00.000000000 Z
+date: 2020-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: delivery_boy
+  name: concurrent-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: dry-configurable
   requirement: !ruby/object:Gem::Requirement
@@ -84,28 +84,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.3'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: rdkafka
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 0.6.0
 - !ruby/object:Gem::Dependency
   name: zeitwerk
   requirement: !ruby/object:Gem::Requirement
@@ -128,38 +128,45 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".coditsu/ci.yml"
+- ".diffend.yml"
 - ".github/FUNDING.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".rspec"
 - ".ruby-gemset"
 - ".ruby-version"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - Gemfile.lock
-- MIT-LICENCE
+- LICENSE
 - README.md
 - certs/mensfeld.pem
 - config/errors.yml
+- docker-compose.yml
 - lib/water_drop.rb
-- lib/water_drop/async_producer.rb
-- lib/water_drop/base_producer.rb
 - lib/water_drop/config.rb
-- lib/water_drop/config_applier.rb
 - lib/water_drop/contracts.rb
 - lib/water_drop/contracts/config.rb
-- lib/water_drop/contracts/message_options.rb
+- lib/water_drop/contracts/message.rb
 - lib/water_drop/errors.rb
+- lib/water_drop/instrumentation.rb
 - lib/water_drop/instrumentation/monitor.rb
 - lib/water_drop/instrumentation/stdout_listener.rb
-- lib/water_drop/sync_producer.rb
+- lib/water_drop/producer.rb
+- lib/water_drop/producer/async.rb
+- lib/water_drop/producer/buffer.rb
+- lib/water_drop/producer/builder.rb
+- lib/water_drop/producer/dummy_client.rb
+- lib/water_drop/producer/statistics_decorator.rb
+- lib/water_drop/producer/status.rb
+- lib/water_drop/producer/sync.rb
 - lib/water_drop/version.rb
 - lib/waterdrop.rb
-- log/.keep
+- log/.gitkeep
 - waterdrop.gemspec
 homepage: https://github.com/karafka/waterdrop
 licenses:
-- MIT
+- LGPL-3.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -172,11 +179,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.5.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Kafka messaging made easy!

data/.travis.yml

@@ -1,35 +0,0 @@
-services:
-  - docker
-
-dist: trusty
-cache: bundler
-
-git:
-  depth: false
-
-test: &test
-  stage: Test
-  language: ruby
-  before_install:
-    - gem install bundler
-    - gem update --system
-  script: bundle exec rspec
-
-jobs:
-  include:
-    - <<: *test
-      rvm: 2.6.5
-    - <<: *test
-      rvm: 2.5.5
-
-    - stage: coditsu
-      language: ruby
-      rvm: 2.6.5
-      before_install:
-        - gem update --system
-        - gem install bundler
-      script: \curl -sSL https://api.coditsu.io/run/ci | bash
-
-stages:
-  - coditsu
-  - test

data/MIT-LICENCE

@@ -1,18 +0,0 @@
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/lib/water_drop/async_producer.rb

@@ -1,26 +0,0 @@
-# frozen_string_literal: true
-
-# WaterDrop library
-module WaterDrop
-  # Async producer for messages
-  class AsyncProducer < BaseProducer
-    # Performs message delivery using deliver_async method
-    # @param message [String] message that we want to send to Kafka
-    # @param options [Hash] options (including topic) for producer
-    # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
-    #   somehow invalid and we cannot perform delivery because of that
-    def self.call(message, options)
-      attempts_count ||= 0
-      attempts_count += 1
-
-      validate!(options)
-      return unless WaterDrop.config.deliver
-
-      d_method = WaterDrop.config.raise_on_buffer_overflow ? :deliver_async! : :deliver_async
-
-      DeliveryBoy.send(d_method, message, options)
-    rescue Kafka::Error => e
-      graceful_attempt?(attempts_count, message, options, e) ? retry : raise(e)
-    end
-  end
-end

data/lib/water_drop/base_producer.rb

@@ -1,57 +0,0 @@
-# frozen_string_literal: true
-
-module WaterDrop
-  # Base messages producer that contains all the logic that is exactly the same for both
-  # sync and async producers
-  class BaseProducer
-    # Contract for checking the correctness of the provided data that someone wants to
-    # dispatch to Kafka
-    SCHEMA = Contracts::MessageOptions.new.freeze
-
-    private_constant :SCHEMA
-
-    class << self
-      private
-
-      # Runs the message options validations and raises an error if anything is invalid
-      # @param options [Hash] hash that we want to validate
-      # @raise [WaterDrop::Errors::InvalidMessageOptions] raised when message options are
-      #   somehow invalid and we cannot perform delivery because of that
-      def validate!(options)
-        validation_result = SCHEMA.call(options)
-        return true if validation_result.success?
-
-        raise Errors::InvalidMessageOptions, validation_result.errors
-      end
-
-      # Upon failed delivery, we may try to resend a message depending on the attempt number
-      #   or re-raise an error if we're unable to do that after given number of retries
-      #   This method checks that and also instruments errors and retries for the delivery
-      # @param attempts_count [Integer] number of attempt (starting from 1) for the delivery
-      # @param message [String] message that we want to send to Kafka
-      # @param options [Hash] options (including topic) for producer
-      # @param error [Kafka::Error] error that occurred
-      # @return [Boolean] true if this is a graceful attempt and we can retry or false it this
-      #   was the final one and we should deal with the fact, that we cannot deliver a given
-      #   message
-      def graceful_attempt?(attempts_count, message, options, error)
-        scope = "#{to_s.split('::').last.sub('Producer', '_producer').downcase}.call"
-        payload = {
-          caller: self,
-          message: message,
-          options: options,
-          error: error,
-          attempts_count: attempts_count
-        }
-
-        if attempts_count > WaterDrop.config.kafka.max_retries
-          WaterDrop.monitor.instrument("#{scope}.error", payload)
-          false
-        else
-          WaterDrop.monitor.instrument("#{scope}.retry", payload)
-          true
-        end
-      end
-    end
-  end
-end