waterdrop 2.6.8 → 2.6.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 58c8488102aefb36da6ba9694231b5d3ee72042fb46e7ba807197e35b505628a
-  data.tar.gz: 6eab9d007458dfee8a7969e04ec216b915d20c44207319b28a73bda01775b88a
+  metadata.gz: f8108bc92f548809ab7220d8d36a03ceb72e283dd651d2d2fc0629bb81a36fc8
+  data.tar.gz: a9bbb98af4e8b1425082fd5816e1441c91a263acabbeb987b75d7d598fe72d8c
 SHA512:
-  metadata.gz: 26e72172752ed70452fced5fc0aad2576ad5ae75c9d41a8a2c022acc19e1d41e4e4e7f50e7781c2e711e749f343ba572edaceca3e635ba2a82c549ee35da774c
-  data.tar.gz: 05460540c2aa4e0c5a8827003a77057e52c9d6a08ae0f5bace989127b6c234c1b5d8fef82949e6e0acac9c4d981a280b828ca3322a34baf70f6359aa434ba31b
+  metadata.gz: 3e11745fc2986fcdc1ac8c5b4a8f4eeeb71c812b961eb296585e71d12d3601e9fcf124ad02d219db049f774a6417f6a293ce9cf5ce282ff5b70a05067fd310ec
+  data.tar.gz: 9488e3cf3753077bf790dbbd815889214dc0e66a4de871e460dd25a287ee3df309a14eed28969bf41cc77c76b4b3eaa64cd2c44a8f8f1556d78828cdf0a08659

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # WaterDrop changelog
 
+## 2.6.10 (2023-10-24)
+- [Improvement] Introduce `message.purged` event to indicate that a message that was not delivered to Kafka was purged. This most of the time refers to messages that were part of a transaction and were not yet dispatched to Kafka. It always means, that given message was not delivered but in case of transactions it is expected. In case of non-transactional it usually means `#purge` usage or exceeding `message.timeout.ms` so `librdkafka` removes this message from its internal queue. Non-transactional producers do **not** use this and pipe purges to `error.occurred`.
+- [Fix] Fix a case where `message.acknowledged` would not have `caller` key.
+- [Fix] Fix a bug where critical errors (like `IRB::Abort`) would not abort the ongoing transaction.
+
+## 2.6.9 (2023-10-23)
+- [Improvement] Introduce a `transaction.finished` event to indicate that transaction has finished whether it was aborted or committed.
+- [Improvement] Use `transaction.committed` event to indicate that transaction has been committed.
+
 ## 2.6.8 (2023-10-20)
 - **[Feature]** Introduce transactions support.
 - [Improvement] Expand `LoggerListener` to inform about transactions (info level).

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    waterdrop (2.6.8)
+    waterdrop (2.6.10)
       karafka-core (>= 2.2.3, < 3.0.0)
       zeitwerk (~> 2.3)
 

data/lib/waterdrop/clients/rdkafka.rb

@@ -19,6 +19,7 @@ module WaterDrop
           # callbacks manager to make it work
           client.delivery_callback = Instrumentation::Callbacks::Delivery.new(
             producer.id,
+            producer.transactional?,
             producer.config.monitor
           )
 

data/lib/waterdrop/config.rb

@@ -64,9 +64,10 @@ module WaterDrop
     # option [Numeric] how many seconds should we wait with the backoff on queue having space for
     # more messages before re-raising the error.
     setting :wait_timeout_on_queue_full, default: 10
-
+    # option [Numeric] How long to wait before retrying a retryable transaction related error
     setting :wait_backoff_on_transaction_command, default: 0.5
-
+    # option [Numeric] How many times to retry a retryable transaction related error before
+    #   giving up
     setting :max_attempts_on_transaction_command, default: 5
 
     # option [Boolean] should we send messages. Setting this to false can be really useful when

data/lib/waterdrop/instrumentation/callbacks/delivery.rb

@@ -6,19 +6,41 @@ module WaterDrop
       # Creates a callable that we want to run upon each message delivery or failure
       #
       # @note We don't have to provide client_name here as this callback is per client instance
+      #
+      # @note We do not consider `message.purge` as an error for transactional producers, because
+      #   this is a standard behaviour for not yet dispatched messages on aborted transactions.
+      #   We do however still want to instrument it for traceability.
       class Delivery
+        # Error emitted when a message was not yet dispatched and was purged from the queue
+        RD_KAFKA_RESP_PURGE_QUEUE = -152
+
+        # Error emitted when a message was purged while it was dispatched
+        RD_KAFKA_RESP_PURGE_INFLIGHT = -151
+
+        # Errors related to queue purging that is expected in transactions
+        PURGE_ERRORS = [RD_KAFKA_RESP_PURGE_INFLIGHT, RD_KAFKA_RESP_PURGE_QUEUE].freeze
+
+        private_constant :RD_KAFKA_RESP_PURGE_QUEUE, :RD_KAFKA_RESP_PURGE_INFLIGHT, :PURGE_ERRORS
+
         # @param producer_id [String] id of the current producer
+        # @param transactional [Boolean] is this handle for a transactional or regular producer
         # @param monitor [WaterDrop::Instrumentation::Monitor] monitor we are using
-        def initialize(producer_id, monitor)
+        def initialize(producer_id, transactional, monitor)
           @producer_id = producer_id
+          @transactional = transactional
           @monitor = monitor
         end
 
         # Emits delivery details to the monitor
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
         def call(delivery_report)
-          if delivery_report.error.to_i.zero?
+          error_code = delivery_report.error.to_i
+
+          if error_code.zero?
             instrument_acknowledged(delivery_report)
+
+          elsif @transactional && PURGE_ERRORS.include?(error_code)
+            instrument_purged(delivery_report)
           else
             instrument_error(delivery_report)
           end
@@ -27,24 +49,24 @@ module WaterDrop
         private
 
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
-        def instrument_error(delivery_report)
+        def instrument_acknowledged(delivery_report)
           @monitor.instrument(
-            'error.occurred',
+            'message.acknowledged',
             caller: self,
-            error: ::Rdkafka::RdkafkaError.new(delivery_report.error),
             producer_id: @producer_id,
             offset: delivery_report.offset,
             partition: delivery_report.partition,
             topic: delivery_report.topic_name,
-            delivery_report: delivery_report,
-            type: 'librdkafka.dispatch_error'
+            delivery_report: delivery_report
           )
         end
 
         # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
-        def instrument_acknowledged(delivery_report)
+        def instrument_purged(delivery_report)
           @monitor.instrument(
-            'message.acknowledged',
+            'message.purged',
+            caller: self,
+            error: build_error(delivery_report),
             producer_id: @producer_id,
             offset: delivery_report.offset,
             partition: delivery_report.partition,
@@ -52,6 +74,28 @@ module WaterDrop
             delivery_report: delivery_report
           )
         end
+
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        def instrument_error(delivery_report)
+          @monitor.instrument(
+            'error.occurred',
+            caller: self,
+            error: build_error(delivery_report),
+            producer_id: @producer_id,
+            offset: delivery_report.offset,
+            partition: delivery_report.partition,
+            topic: delivery_report.topic_name,
+            delivery_report: delivery_report,
+            type: 'librdkafka.dispatch_error'
+          )
+        end
+
+        # Builds appropriate rdkafka error
+        # @param delivery_report [Rdkafka::Producer::DeliveryReport] delivery report
+        # @return [::Rdkafka::RdkafkaError]
+        def build_error(delivery_report)
+          ::Rdkafka::RdkafkaError.new(delivery_report.error)
+        end
       end
     end
   end

data/lib/waterdrop/instrumentation/logger_listener.rb

@@ -145,6 +145,11 @@ module WaterDrop
         info(event, 'Committing transaction')
       end
 
+      # @param event [Dry::Events::Event] event that happened with the details
+      def on_transaction_finished(event)
+        info(event, 'Processing transaction')
+      end
+
       private
 
       # @return [Boolean] should we report the messages details in the debug mode.

data/lib/waterdrop/instrumentation/notifications.rb

@@ -12,6 +12,7 @@ module WaterDrop
         message.produced_async
         message.produced_sync
         message.acknowledged
+        message.purged
         message.buffered
 
         messages.produced_async
@@ -21,6 +22,7 @@ module WaterDrop
         transaction.started
         transaction.committed
         transaction.aborted
+        transaction.finished
 
         buffer.flushed_async
         buffer.flushed_sync

data/lib/waterdrop/producer/transactions.rb

@@ -49,7 +49,7 @@ module WaterDrop
         return yield if @transaction_mutex.owned?
 
         @transaction_mutex.synchronize do
-          transactional_instrument(:committed) do
+          transactional_instrument(:finished) do
             with_transactional_error_handling(:begin) do
               transactional_instrument(:started) { client.begin_transaction }
             end
@@ -65,11 +65,16 @@ module WaterDrop
             commit || raise(WaterDrop::Errors::AbortTransaction)
 
             with_transactional_error_handling(:commit) do
-              client.commit_transaction
+              transactional_instrument(:committed) { client.commit_transaction }
             end
 
             result
-          rescue StandardError => e
+          # We need to handle any interrupt including critical in order not to have the transaction
+          # running. This will also handle things like `IRB::Abort`
+          #
+          # rubocop:disable Lint/RescueException
+          rescue Exception => e
+            # rubocop:enable Lint/RescueException
             with_transactional_error_handling(:abort) do
               transactional_instrument(:aborted) { client.abort_transaction }
             end

data/lib/waterdrop/version.rb

@@ -3,5 +3,5 @@
 # WaterDrop library
 module WaterDrop
   # Current WaterDrop version
-  VERSION = '2.6.8'
+  VERSION = '2.6.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: waterdrop
 version: !ruby/object:Gem::Version
-  version: 2.6.8
+  version: 2.6.10
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-10-20 00:00:00.000000000 Z
+date: 2023-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core