karafka 2.4.16 → 2.4.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8510d261a0722b764aff2c59e214bb2b87481ce606488e8f1323c4bf29ee4528
-  data.tar.gz: 4574f71706749ec5effa830b9c04a6e3072c880220a08fa33f81648f7d3c79be
+  metadata.gz: a26065fae7fe2d8f878b56e497d3551f3f0277b27ed3356b59fd478f017b4144
+  data.tar.gz: 356d0e078a32f0b22dc7d06d0f0e896d17b55e7a52866acf92933aeb6a611548
 SHA512:
-  metadata.gz: 37c43b9634b43e6608fa7f2ab9320d49d00b2582c211095968d3200412029e0301d17556d31be2e24d8401457fd660473cc46adc4d5835de968a2eba9c51fbef
-  data.tar.gz: 15d2e2dbd7ee99ab1c726a47a76959f1e3d877cd723f084550711fd9ba4662a30eb39af26f1354847ab906aaec9580e0cde5f7858b0cda521f5f39844195a3c3
+  metadata.gz: 1043bb3ff799cc5294de4936552f4dcc3a78f9aabca275fd35259c87dfdc257cc63a778722d833c8a1f3e9449084e6266e6cd507d396cd92457b8e6078ced631
+  data.tar.gz: 57fe56b280f460be5f3fe2a734587d75ad08e25ae96363da68d5791fe62b6427402de400e6cae2e38660bbcefdb314be45d0299d15c50131a1ebe15f44c23c96

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Karafka Framework Changelog
 
+## 2.4.17 (2025-01-15)
+- [Enhancement] Clean message key and headers when cleaning messages via the cleaner API (Pro).
+- [Enhancement] Allow for setting `metadata: false` in the cleaner API for granular cleaning control (Pro)
+- [Enhancement] Instrument successful transaction via `consumer.consuming.transaction` event (Pro).
+
 ## 2.4.16 (2024-12-27)
 - [Enhancement] Improve post-rebalance revocation messages filtering.
 - [Enhancement] Introduce `Consumer#wrap` for connection pooling management and other wrapped operations.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.4.16)
+    karafka (2.4.17)
       base64 (~> 0.2)
       karafka-core (>= 2.4.4, < 2.5.0)
       karafka-rdkafka (>= 0.17.2)
@@ -123,4 +123,4 @@ DEPENDENCIES
   stringio
 
 BUNDLED WITH
-   2.4.22
+   2.6.2

data/LICENSE

@@ -6,8 +6,13 @@ for license text.
 
 Karafka has also commercial-friendly license, commercial support and commercial components.
 
-All of the commercial components are present in the lib/karafka/pro directory of this repository
-and their usage requires commercial license agreement.
+All of the commercial components are present in the:
+
+- `lib/karafka/pro`
+- `spec/lib/karafka/pro`
+- `spec/integrations/pro`
+
+directories of this repository and their usage requires commercial license agreement.
 
 By sending a pull request to the pro components, you are agreeing to transfer the copyright of your
 code to Maciej Mensfeld.

data/lib/karafka/base_consumer.rb

@@ -12,7 +12,7 @@ module Karafka
 
     extend Forwardable
 
-    def_delegators :@coordinator, :topic, :partition, :eofed?
+    def_delegators :@coordinator, :topic, :partition, :eofed?, :seek_offset, :seek_offset=
 
     def_delegators :producer, :produce_async, :produce_sync, :produce_many_async,
                    :produce_many_sync
@@ -109,7 +109,7 @@ module Karafka
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.consume.error'
       )
     end
@@ -129,7 +129,7 @@ module Karafka
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.after_consume.error'
       )
 
@@ -149,7 +149,7 @@ module Karafka
         'error.occurred',
         error: e,
         caller: self,
-        seek_offset: coordinator.seek_offset,
+        seek_offset: seek_offset,
         type: 'consumer.eofed.error'
       )
     end
@@ -345,7 +345,7 @@ module Karafka
     # @note Please note, that if you are seeking to a time offset, getting the offset is blocking
     def seek(offset, manual_seek = true, reset_offset: false)
       coordinator.manual_seek if manual_seek
-      coordinator.seek_offset = nil if reset_offset
+      self.seek_offset = nil if reset_offset
 
       message = Karafka::Messages::Seek.new(
         topic.name,
@@ -397,7 +397,7 @@ module Karafka
     # Pauses the processing from the last offset to retry on given message
     # @private
     def retry_after_pause
-      pause(coordinator.seek_offset, nil, false)
+      pause(seek_offset, nil, false)
 
       # Instrumentation needs to run **after** `#pause` invocation because we rely on the states
       # set by `#pause`
@@ -406,7 +406,7 @@ module Karafka
         caller: self,
         topic: topic.name,
         partition: partition,
-        offset: coordinator.seek_offset,
+        offset: seek_offset,
         timeout: coordinator.pause_tracker.current_timeout,
         attempt: attempt
       )

data/lib/karafka/instrumentation/notifications.rb

@@ -57,6 +57,7 @@ module Karafka
         consumer.consuming.pause
         consumer.consuming.retry
         consumer.consuming.seek
+        consumer.consuming.transaction
 
         consumer.before_schedule_idle
         consumer.idle

data/lib/karafka/pro/active_job/consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/active_job/dispatcher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/active_job/job_options_contract.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/base_consumer.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/errors.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/messages/message.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -29,15 +21,25 @@ module Karafka
             @raw_payload == false
           end
 
-          # Cleans the message payload and removes the deserialized data references
+          # Cleans the message payload, headers, key and removes the deserialized data references
           # This is useful when working with big messages that take a lot of space.
           #
           # After the message content is no longer needed, it can be removed so it does not consume
           # space anymore.
-          def clean!
+          #
+          # @param metadata [Boolean] should we also clean metadata alongside the payload. This can
+          #   be useful when working with iterator and other things that may require only metadata
+          #   available, while not payload. `true` by default.
+          #
+          # @note Cleaning of message means we also clean its metadata (headers and key)
+          # @note Metadata cleaning (headers and key) can be disabled by setting the `metadata`
+          #   argument to `false`.
+          def clean!(metadata: true)
             @deserialized = false
             @raw_payload = false
             @payload = nil
+
+            @metadata.clean! if metadata
           end
         end
       end

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/cleaner/messages/metadata.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner messages components related enhancements
+      module Messages
+        # Extensions to the message metadata that allow for granular memory control on a per
+        # message basis
+        module Metadata
+          # @return [Object] deserialized key. By default in the raw string format.
+          def key
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Object] deserialized headers. By default its a hash with keys and payload
+          #   being strings
+          def headers
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Boolean] true if the message metadata has been cleaned
+          def cleaned?
+            raw_headers == false
+          end
+
+          # Cleans the headers and key
+          def clean!
+            self.raw_headers = false
+            self.raw_key = false
+            @key = nil
+            @headers = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro
@@ -28,6 +20,7 @@ module Karafka
         # @param _config [Karafka::Core::Configurable::Node] root node config
         def pre_setup(_config)
           ::Karafka::Messages::Message.prepend(Messages::Message)
+          ::Karafka::Messages::Metadata.prepend(Messages::Metadata)
           ::Karafka::Messages::Messages.prepend(Messages::Messages)
         end
 

data/lib/karafka/pro/connection/manager.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/connection/multiplexing/listener.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/contracts/base.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/contracts/server_cli_options.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/cipher.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/contracts/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/errors.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/messages/middleware.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/messages/parser.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption/setup/config.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/encryption.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/instrumentation/performance_tracker.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator/expander.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator/tpl_builder.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro

data/lib/karafka/pro/iterator.rb

@@ -1,15 +1,7 @@
 # frozen_string_literal: true
 
-# This Karafka component is a Pro component under a commercial license.
-# This Karafka component is NOT licensed under LGPL.
-#
-# All of the commercial components are present in the lib/karafka/pro directory of this
-# repository and their usage requires commercial license agreement.
-#
-# Karafka has also commercial-friendly license, commercial support and commercial components.
-#
-# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
-# your code to Maciej Mensfeld.
+# This code is part of Karafka Pro, a commercial component not licensed under LGPL.
+# See LICENSE for details.
 
 module Karafka
   module Pro