waterdrop 2.0.7 → 2.6.14

data/lib/waterdrop/producer/transactions.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  class Producer
+    # Transactions related producer functionalities
+    module Transactions
+      # Contract to validate that input for transactional offset storage is correct
+      CONTRACT = Contracts::TransactionalOffset.new
+
+      private_constant :CONTRACT
+
+      # Creates a transaction.
+      #
+      # Karafka transactions work in a similar manner to SQL db transactions though there are some
+      # crucial differences. When you start a transaction, all messages produced during it will
+      # be delivered together or will fail together. The difference is, that messages from within
+      # a single transaction can be delivered and will have a delivery handle but will be then
+      # compacted prior to moving the LSO forward. This means, that not every delivery handle for
+      # async dispatches will emit a queue purge error. None for sync as the delivery has happened
+      # but they will never be visible by the transactional consumers.
+      #
+      # Transactions **are** thread-safe however they lock a mutex. This means, that for
+      # high-throughput transactional messages production in multiple threads
+      # (for example in Karafka), it may be much better to use few instances that can work in
+      # parallel.
+      #
+      # Please note, that if a producer is configured as transactional, it **cannot** produce
+      # messages outside of transactions, that is why by default all dispatches will be wrapped
+      # with a transaction. One transaction per single dispatch and for `produce_many` it will be
+      # a single transaction wrapping all messages dispatches (not one per message).
+      #
+      # @return Block result
+      #
+      # @example Simple transaction
+      #   producer.transaction do
+      #     producer.produce_async(topic: 'topic', payload: 'data')
+      #   end
+      #
+      # @example Aborted transaction - messages producer won't be visible by consumers
+      #   producer.transaction do
+      #     producer.produce_sync(topic: 'topic', payload: 'data')
+      #     throw(:abort)
+      #   end
+      #
+      # @example Use block result last handler to wait on all messages ack
+      #   handler = producer.transaction do
+      #               producer.produce_async(topic: 'topic', payload: 'data')
+      #             end
+      #
+      #   handler.wait
+      def transaction
+        # This will safely allow us to support one operation transactions so a transactional
+        # producer can work without the transactional block if needed
+        return yield if @transaction_mutex.owned?
+
+        @transaction_mutex.synchronize do
+          transactional_instrument(:finished) do
+            with_transactional_error_handling(:begin) do
+              transactional_instrument(:started) { client.begin_transaction }
+            end
+
+            result = nil
+            commit = false
+
+            catch(:abort) do
+              result = yield
+              commit = true
+            end
+
+            commit || raise(WaterDrop::Errors::AbortTransaction)
+
+            with_transactional_error_handling(:commit) do
+              transactional_instrument(:committed) { client.commit_transaction }
+            end
+
+            result
+          # We need to handle any interrupt including critical in order not to have the transaction
+          # running. This will also handle things like `IRB::Abort`
+          #
+          # rubocop:disable Lint/RescueException
+          rescue Exception => e
+            # rubocop:enable Lint/RescueException
+            with_transactional_error_handling(:abort) do
+              transactional_instrument(:aborted) { client.abort_transaction }
+            end
+
+            raise unless e.is_a?(WaterDrop::Errors::AbortTransaction)
+          end
+        end
+      end
+
+      # @return [Boolean] Is this producer a transactional one
+      def transactional?
+        return @transactional if instance_variable_defined?(:'@transactional')
+
+        @transactional = config.kafka.to_h.key?(:'transactional.id')
+      end
+
+      # Marks given message as consumed inside of a transaction.
+      #
+      # @param consumer [#consumer_group_metadata_pointer] any consumer from which we can obtain
+      #   the librdkafka consumer group metadata pointer
+      # @param message [Karafka::Messages::Message] karafka message
+      # @param offset_metadata [String] offset metadata or nil if none
+      def transaction_mark_as_consumed(consumer, message, offset_metadata = nil)
+        raise Errors::TransactionRequiredError unless @transaction_mutex.owned?
+
+        CONTRACT.validate!(
+          {
+            consumer: consumer,
+            message: message,
+            offset_metadata: offset_metadata
+          },
+          Errors::TransactionalOffsetInvalidError
+        )
+
+        details = { message: message, offset_metadata: offset_metadata }
+
+        transactional_instrument(:marked_as_consumed, details) do
+          tpl = Rdkafka::Consumer::TopicPartitionList.new
+          partition = Rdkafka::Consumer::Partition.new(
+            message.partition,
+            # +1 because this is next offset from which we will start processing from
+            message.offset + 1,
+            0,
+            offset_metadata
+          )
+
+          tpl.add_topic_and_partitions_with_offsets(message.topic, [partition])
+
+          with_transactional_error_handling(:store_offset) do
+            client.send_offsets_to_transaction(
+              consumer,
+              tpl,
+              # This setting is at the moment in seconds and we require ms
+              @config.max_wait_timeout * 1_000
+            )
+          end
+        end
+      end
+
+      private
+
+      # Runs provided code with a transaction wrapper if transactions are enabled.
+      # This allows us to simplify the async and sync batch dispatchers because we can ensure that
+      # their internal dispatches will be wrapped only with a single transaction and not
+      # a transaction per message
+      # @param block [Proc] code we want to run
+      def with_transaction_if_transactional(&block)
+        transactional? ? transaction(&block) : yield
+      end
+
+      # Instruments the transactional operation with producer id
+      #
+      # @param key [Symbol] transaction operation key
+      # @param details [Hash] additional instrumentation details
+      # @param block [Proc] block to run inside the instrumentation or nothing if not given
+      def transactional_instrument(key, details = EMPTY_HASH, &block)
+        @monitor.instrument("transaction.#{key}", details.merge(producer_id: id), &block)
+      end
+
+      # Error handling for transactional operations is a bit special. There are three types of
+      # errors coming from librdkafka:
+      #   - retryable - indicates that a given operation (like offset commit) can be retried after
+      #     a backoff and that is should be operating later as expected. We try to retry those
+      #     few times before finally failing.
+      #   - fatal - errors that will not recover no matter what (for example being fenced out)
+      #   - abortable - error from which we cannot recover but for which we should abort the
+      #     current transaction.
+      #
+      # The code below handles this logic also publishing the appropriate notifications via our
+      # notifications pipeline.
+      #
+      # @param action [Symbol] action type
+      # @param allow_abortable [Boolean] should we allow for the abortable flow. This is set to
+      #   false internally to prevent attempts to abort from failed abort operations
+      def with_transactional_error_handling(action, allow_abortable: true)
+        attempt ||= 0
+        attempt += 1
+
+        yield
+      rescue ::Rdkafka::RdkafkaError => e
+        # Decide if there is a chance to retry given error
+        do_retry = e.retryable? && attempt < config.max_attempts_on_transaction_command
+
+        @monitor.instrument(
+          'error.occurred',
+          producer_id: id,
+          caller: self,
+          error: e,
+          type: "transaction.#{action}",
+          retry: do_retry,
+          attempt: attempt
+        )
+
+        raise if e.fatal?
+
+        if do_retry
+          # Backoff more and more before retries
+          sleep(config.wait_backoff_on_transaction_command * attempt)
+
+          retry
+        end
+
+        if e.abortable? && allow_abortable
+          # Always attempt to abort but if aborting fails with an abortable error, do not attempt
+          # to abort from abort as this could create an infinite loop
+          with_transactional_error_handling(:abort, allow_abortable: false) do
+            transactional_instrument(:aborted) { client.abort_transaction }
+          end
+
+          raise
+        end
+
+        raise
+      end
+    end
+  end
+end

data/lib/waterdrop/producer.rb

@@ -0,0 +1,324 @@
+# frozen_string_literal: true
+
+module WaterDrop
+  # Main WaterDrop messages producer
+  class Producer
+    extend Forwardable
+    include Sync
+    include Async
+    include Buffer
+    include Transactions
+    include ::Karafka::Core::Helpers::Time
+
+    # Which of the inline flow errors do we want to intercept and re-bind
+    SUPPORTED_FLOW_ERRORS = [
+      Rdkafka::RdkafkaError,
+      Rdkafka::Producer::DeliveryHandle::WaitTimeoutError
+    ].freeze
+
+    # Empty has to save on memory allocations
+    EMPTY_HASH = {}.freeze
+
+    private_constant :SUPPORTED_FLOW_ERRORS, :EMPTY_HASH
+
+    def_delegators :config, :middleware
+
+    # @return [String] uuid of the current producer
+    attr_reader :id
+    # @return [Status] producer status object
+    attr_reader :status
+    # @return [Array] internal messages buffer
+    attr_reader :messages
+    # @return [Object] monitor we want to use
+    attr_reader :monitor
+    # @return [Object] dry-configurable config object
+    attr_reader :config
+
+    # Creates a not-yet-configured instance of the producer
+    # @param block [Proc] configuration block
+    # @return [Producer] producer instance
+    def initialize(&block)
+      @operations_in_progress = Helpers::Counter.new
+      @buffer_mutex = Mutex.new
+      @connecting_mutex = Mutex.new
+      @operating_mutex = Mutex.new
+      @transaction_mutex = Mutex.new
+
+      @status = Status.new
+      @messages = []
+
+      return unless block
+
+      setup(&block)
+    end
+
+    # Sets up the whole configuration and initializes all that is needed
+    # @param block [Block] configuration block
+    def setup(&block)
+      raise Errors::ProducerAlreadyConfiguredError, id unless @status.initial?
+
+      @config = Config
+                .new
+                .setup(&block)
+                .config
+
+      @id = @config.id
+      @monitor = @config.monitor
+      @contract = Contracts::Message.new(max_payload_size: @config.max_payload_size)
+      @status.configured!
+    end
+
+    # @return [Rdkafka::Producer] raw rdkafka producer
+    # @note Client is lazy initialized, keeping in mind also the fact of a potential fork that
+    #   can happen any time.
+    # @note It is not recommended to fork a producer that is already in use so in case of
+    #   bootstrapping a cluster, it's much better to fork configured but not used producers
+    def client
+      return @client if @client && @pid == Process.pid
+
+      # Don't allow to obtain a client reference for a producer that was not configured
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+
+      @connecting_mutex.synchronize do
+        return @client if @client && @pid == Process.pid
+
+        # We undefine all the finalizers, in case it was a fork, so the finalizers from the parent
+        # process don't leak
+        ObjectSpace.undefine_finalizer(id)
+
+        # We should raise an error when trying to use a producer with client from a fork. Always.
+        if @client
+          # We need to reset the client, otherwise there might be attempt to close the parent
+          # client
+          @client = nil
+          raise Errors::ProducerUsedInParentProcess, Process.pid
+        end
+
+        # Finalizer tracking is needed for handling shutdowns gracefully.
+        # I don't expect everyone to remember about closing all the producers all the time, thus
+        # this approach is better. Although it is still worth keeping in mind, that this will
+        # block GC from removing a no longer used producer unless closed properly but at least
+        # won't crash the VM upon closing the process
+        ObjectSpace.define_finalizer(id, proc { close })
+
+        @pid = Process.pid
+        @client = Builder.new.call(self, @config)
+
+        # Register statistics runner for this particular type of callbacks
+        ::Karafka::Core::Instrumentation.statistics_callbacks.add(
+          @id,
+          Instrumentation::Callbacks::Statistics.new(@id, @client.name, @config.monitor)
+        )
+
+        # Register error tracking callback
+        ::Karafka::Core::Instrumentation.error_callbacks.add(
+          @id,
+          Instrumentation::Callbacks::Error.new(@id, @client.name, @config.monitor)
+        )
+
+        @status.connected!
+        @monitor.instrument('producer.connected', producer_id: id)
+      end
+
+      @client
+    end
+
+    # Fetches and caches the partition count of a topic
+    #
+    # @param topic [String] topic for which we want to get the number of partitions
+    # @return [Integer] number of partitions of the requested topic or -1 if number could not be
+    #   retrieved.
+    #
+    # @note It uses the underlying `rdkafka-ruby` partition count fetch and cache.
+    def partition_count(topic)
+      client.partition_count(topic.to_s)
+    end
+
+    # Purges data from both the buffer queue as well as the librdkafka queue.
+    #
+    # @note This is an operation that can cause data loss. Keep that in mind. It will not only
+    #   purge the internal WaterDrop buffer but will also purge the librdkafka queue as well as
+    #   will cancel any outgoing messages dispatches.
+    def purge
+      @monitor.instrument('buffer.purged', producer_id: id) do
+        @buffer_mutex.synchronize do
+          @messages = []
+        end
+
+        @client.purge
+      end
+    end
+
+    # Flushes the buffers in a sync way and closes the producer
+    # @param force [Boolean] should we force closing even with outstanding messages after the
+    #   max wait timeout
+    def close(force: false)
+      @operating_mutex.synchronize do
+        return unless @status.active?
+
+        @monitor.instrument(
+          'producer.closed',
+          producer_id: id
+        ) do
+          @status.closing!
+          @monitor.instrument('producer.closing', producer_id: id)
+
+          # No need for auto-gc if everything got closed by us
+          # This should be used only in case a producer was not closed properly and forgotten
+          ObjectSpace.undefine_finalizer(id)
+
+          # We save this thread id because we need to bypass the activity verification on the
+          # producer for final flush of buffers.
+          @closing_thread_id = Thread.current.object_id
+
+          # Wait until all the outgoing operations are done. Only when no one is using the
+          # underlying client running operations we can close
+          sleep(0.001) until @operations_in_progress.value.zero?
+
+          # Flush has its own buffer mutex but even if it is blocked, flushing can still happen
+          # as we close the client after the flushing (even if blocked by the mutex)
+          flush(true)
+
+          # We should not close the client in several threads the same time
+          # It is safe to run it several times but not exactly the same moment
+          # We also mark it as closed only if it was connected, if not, it would trigger a new
+          # connection that anyhow would be immediately closed
+          if @client
+            # Why do we trigger it early instead of just having `#close` do it?
+            # The linger.ms time will be ignored for the duration of the call,
+            # queued messages will be sent to the broker as soon as possible.
+            begin
+              # `max_wait_timeout` is in seconds at the moment
+              @client.flush(@config.max_wait_timeout * 1_000) unless @client.closed?
+            # We can safely ignore timeouts here because any left outstanding requests
+            # will anyhow force wait on close if not forced.
+            # If forced, we will purge the queue and just close
+            rescue ::Rdkafka::RdkafkaError, Rdkafka::AbstractHandle::WaitTimeoutError
+              nil
+            ensure
+              # Purge fully the local queue in case of a forceful shutdown just to be sure, that
+              # there are no dangling messages. In case flush was successful, there should be
+              # none but we do it just in case it timed out
+              purge if force
+            end
+
+            @client.close
+
+            @client = nil
+          end
+
+          # Remove callbacks runners that were registered
+          ::Karafka::Core::Instrumentation.statistics_callbacks.delete(@id)
+          ::Karafka::Core::Instrumentation.error_callbacks.delete(@id)
+
+          @status.closed!
+        end
+      end
+    end
+
+    # Closes the producer with forced close after timeout, purging any outgoing data
+    def close!
+      close(force: true)
+    end
+
+    private
+
+    # Ensures that we don't run any operations when the producer is not configured or when it
+    # was already closed
+    def ensure_active!
+      return if @status.active?
+      return if @status.closing? && @operating_mutex.owned?
+
+      raise Errors::ProducerNotConfiguredError, id if @status.initial?
+      raise Errors::ProducerClosedError, id if @status.closing?
+      raise Errors::ProducerClosedError, id if @status.closed?
+
+      # This should never happen
+      raise Errors::StatusInvalidError, [id, @status.to_s]
+    end
+
+    # Ensures that the message we want to send out to Kafka is actually valid and that it can be
+    # sent there
+    # @param message [Hash] message we want to send
+    # @raise [Karafka::Errors::MessageInvalidError]
+    def validate_message!(message)
+      @contract.validate!(message, Errors::MessageInvalidError)
+    end
+
+    # Waits on a given handler
+    #
+    # @param handler [Rdkafka::Producer::DeliveryHandle]
+    def wait(handler)
+      handler.wait(
+        max_wait_timeout: @config.max_wait_timeout,
+        wait_timeout: @config.wait_timeout
+      )
+    end
+
+    # Runs the client produce method with a given message
+    #
+    # @param message [Hash] message we want to send
+    def produce(message)
+      produce_time ||= monotonic_now
+
+      # This can happen only during flushing on closing, in case like this we don't have to
+      # synchronize because we already own the lock
+      if @operating_mutex.owned?
+        @operations_in_progress.increment
+      else
+        @operating_mutex.synchronize { @operations_in_progress.increment }
+        ensure_active!
+      end
+
+      # In case someone defines topic as a symbol, we need to convert it into a string as
+      # librdkafka does not accept symbols
+      message = message.merge(topic: message[:topic].to_s) if message[:topic].is_a?(Symbol)
+
+      if transactional?
+        transaction { client.produce(**message) }
+      else
+        client.produce(**message)
+      end
+    rescue SUPPORTED_FLOW_ERRORS.first => e
+      # Unless we want to wait and retry and it's a full queue, we raise normally
+      raise unless @config.wait_on_queue_full
+      raise unless e.code == :queue_full
+      # If we're running for longer than the timeout, we need to re-raise the queue full.
+      # This will prevent from situation where cluster is down forever and we just retry and retry
+      # in an infinite loop, effectively hanging the processing
+      raise unless monotonic_now - produce_time < @config.wait_timeout_on_queue_full * 1_000
+
+      label = caller_locations(2, 1)[0].label.split(' ').last
+
+      # We use this syntax here because we want to preserve the original `#cause` when we
+      # instrument the error and there is no way to manually assign `#cause` value. We want to keep
+      # the original cause to maintain the same API across all the errors dispatched to the
+      # notifications pipeline.
+      begin
+        raise Errors::ProduceError, e.inspect
+      rescue Errors::ProduceError => e
+        # We want to instrument on this event even when we restart it.
+        # The reason is simple: instrumentation and visibility.
+        # We can recover from this, but despite that we should be able to instrument this.
+        # If this type of event happens too often, it may indicate that the buffer settings are not
+        # well configured.
+        @monitor.instrument(
+          'error.occurred',
+          producer_id: id,
+          message: message,
+          error: e,
+          type: "message.#{label}"
+        )
+
+        # We do not poll the producer because polling happens in a background thread
+        # It also should not be a frequent case (queue full), hence it's ok to just throttle.
+        sleep @config.wait_backoff_on_queue_full
+      end
+
+      @operations_in_progress.decrement
+      retry
+    ensure
+      @operations_in_progress.decrement
+    end
+  end
+end

data/lib/{water_drop → waterdrop}/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.0.7'
+  VERSION = '2.6.14'
 end

data/lib/waterdrop.rb

@@ -1,4 +1,29 @@
 # frozen_string_literal: true
 
-# This file is used as a compatibility step
-require 'water_drop'
+# External components
+# delegate should be removed because we don't need it, we just add it because of ruby-kafka
+%w[
+  forwardable
+  json
+  zeitwerk
+  securerandom
+  karafka-core
+  pathname
+].each { |lib| require lib }
+
+# WaterDrop library
+module WaterDrop
+  class << self
+    # @return [String] root path of this gem
+    def gem_root
+      Pathname.new(File.expand_path('..', __dir__))
+    end
+  end
+end
+
+loader = Zeitwerk::Loader.for_gem
+loader.inflector.inflect('waterdrop' => 'WaterDrop')
+# Do not load vendors instrumentation components. Those need to be required manually if needed
+loader.ignore("#{__dir__}/waterdrop/instrumentation/vendors/**/*.rb")
+loader.setup
+loader.eager_load

data/renovate.json

@@ -0,0 +1,6 @@
+{
+  "$schema": "https://docs.renovatebot.com/renovate-schema.json",
+  "extends": [
+    "config:base"
+  ]
+}

data/waterdrop.gemspec

@@ -3,35 +3,38 @@
 lib = File.expand_path('lib', __dir__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
-require 'water_drop/version'
+require 'waterdrop/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'waterdrop'
   spec.version       = ::WaterDrop::VERSION
   spec.platform      = Gem::Platform::RUBY
   spec.authors       = ['Maciej Mensfeld']
-  spec.email         = %w[maciej@mensfeld.pl]
+  spec.email         = %w[contact@karafka.io]
   spec.homepage      = 'https://karafka.io'
   spec.summary       = 'Kafka messaging made easy!'
   spec.description   = spec.summary
   spec.license       = 'MIT'
 
-  spec.add_dependency 'concurrent-ruby', '>= 1.1'
-  spec.add_dependency 'dry-configurable', '~> 0.13'
-  spec.add_dependency 'dry-monitor', '~> 0.5'
-  spec.add_dependency 'dry-validation', '~> 1.7'
-  spec.add_dependency 'rdkafka', '>= 0.10'
+  spec.add_dependency 'karafka-core', '>= 2.2.3', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
-  spec.required_ruby_version = '>= 2.6.0'
-
   if $PROGRAM_NAME.end_with?('gem')
     spec.signing_key = File.expand_path('~/.ssh/gem-private_key.pem')
   end
 
-  spec.cert_chain    = %w[certs/mensfeld.pem]
+  spec.cert_chain    = %w[certs/cert_chain.pem]
   spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec)/}) }
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = %w[lib]
-  spec.metadata      = { 'source_code_uri' => 'https://github.com/karafka/waterdrop' }
+
+  spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://karafka.io/docs/Changelog-WaterDrop',
+    'bug_tracker_uri' => 'https://github.com/karafka/waterdrop/issues',
+    'source_code_uri' => 'https://github.com/karafka/waterdrop',
+    'documentation_uri' => 'https://karafka.io/docs/#waterdrop',
+    'rubygems_mfa_required' => 'true'
+  }
 end