karafka 2.1.11 → 2.1.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d1e3000ced82b96afc7c4588d20170e0908d3b41305eead2e5e0a850c9a70c4
-  data.tar.gz: 1b16fc0a06f1bfd9bdb1207dd1faac04d3a7f956ccb42edd45ada76fed6dcf09
+  metadata.gz: 262c2bbfa4fb09c4038ce559c71baa6c40b861497dd7ef3b1915ba6b8aa47652
+  data.tar.gz: a34d5fad3bdbd1a58a0938f33c8fd28a5df72cb89293d7835c41cfce09c9e736
 SHA512:
-  metadata.gz: 7e018450fae0ad666bff80ca9875750303946746c20e527200ee569a1a182c3d81bb71b64a0b2bd94b4a35c8cf204326b26752b1d5c0425a1f544ff9b4572323
-  data.tar.gz: 6025b3583179592313540231dd9d00354c30d8af3d49f482fc6c7dbd96359dbc7bbccee8cfa4cf1d14a2f5dfb3a9999b141317c782ec33af7503c4a8e26c616c
+  metadata.gz: 2fd4672fef274f5913b543d9b3a91aef8a8aba59a23e457a61f455cf6095ea8176c6f8e799f65602c7ef0bb15add27a960df70cd99913fe6b5340df9cfff8c31
+  data.tar.gz: 4b890531f2a783c72573bcc2e915cc70b85e09c5e43de76b979fc34ea732f51a2c8ec4f64e91c95293337973103c7eeee89e424d27fa35dc708f935364fa9db3

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka framework changelog
 
+## 2.1.13 (2023-08-28)
+- **[Feature]** Introduce Cleaning API for much better memory management for iterative data processing [Pro].
+- [Enhancement] Automatically free message resources after processed for ActiveJob jobs [Pro]
+- [Enhancement] Free memory used by the raw payload as fast as possible after obtaining it from `karafka-rdkafka`.
+- [Enhancement] Support changing `service_name` in DataDog integration.
+
+## 2.1.12 (2023-08-25)
+- [Fix] Fix a case where DLQ + VP without intermediate marking would mark earlier message then the last one.
+
 ## 2.1.11 (2023-08-23)
 - [Enhancement] Expand the error handling for offset related queries with timeout error retries.
 - [Enhancement] Allow for connection proxy timeouts configuration.

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    karafka (2.1.11)
+    karafka (2.1.13)
       karafka-core (>= 2.1.1, < 2.2.0)
       thor (>= 0.20)
       waterdrop (>= 2.6.6, < 3.0.0)

data/bin/record_rss

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# This script monitors and records the Resident Set Size (RSS) of a process given its PID.
+# The RSS is logged every second to the specified output file until the process terminates.
+#
+# Usage:
+# ./script_name.sh <PID> <OUTPUT_FILE>
+#
+# Arguments:
+# <PID>         - Process ID of the process you want to monitor.
+# <OUTPUT_FILE> - Name of the file where RSS values will be logged.
+#
+# The script first checks if the correct number of arguments are provided.
+# It then verifies if the given PID exists. If it does, it starts recording the RSS.
+# For every iteration, the script fetches the current RSS of the process using the 'ps' command,
+# then appends the RSS value along with a timestamp to the output file.
+# This recording is done every second.
+# The loop stops if the process with the given PID terminates.
+# An informative message is printed out when recording starts and when it stops.
+
+# Check if the correct number of arguments are passed
+if [ "$#" -ne 2 ]; then
+    echo "Usage: $0 <PID> <OUTPUT_FILE>"
+    exit 1
+fi
+
+PID=$1
+OUTPUT_FILE=$2
+
+# Check if the given PID exists
+if ! kill -0 $PID 2>/dev/null; then
+    echo "Error: PID $PID does not exist."
+    exit 1
+fi
+
+# Start recording the RSS
+echo "Recording RSS for PID $PID every second to $OUTPUT_FILE..."
+
+while kill -0 $PID 2>/dev/null; do
+    RSS=$(ps -o rss= -p $PID)
+    if [ -z "$RSS" ]; then
+        echo "Error: Failed to get RSS for PID $PID."
+        exit 1
+    fi
+    TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S')
+    echo "$TIMESTAMP: $RSS KB" >> $OUTPUT_FILE
+    sleep 1
+done
+
+echo "Process $PID has terminated. Stopping recording."

data/lib/karafka/errors.rb

@@ -41,6 +41,9 @@ module Karafka
     # Raised when the license token is not valid
     InvalidLicenseTokenError = Class.new(BaseError)
 
+    # Raised on attempt to deserializer a cleared message
+    MessageClearedError = Class.new(BaseError)
+
     # This should never happen. Please open an issue if it does.
     InvalidCoordinatorStateError = Class.new(BaseError)
 

data/lib/karafka/instrumentation/vendors/datadog/logger_listener.rb

@@ -12,11 +12,14 @@ module Karafka
           include ::Karafka::Core::Configurable
           extend Forwardable
 
-          def_delegators :config, :client
+          def_delegators :config, :client, :service_name
 
           # `Datadog::Tracing` client that we should use to trace stuff
           setting :client
 
+          # @see https://docs.datadoghq.com/tracing/trace_collection/dd_libraries/ruby
+          setting :service_name, default: nil
+
           configure
 
           # Log levels that we use in this particular listener
@@ -44,7 +47,7 @@ module Karafka
           #
           # @param event [Karafka::Core::Monitoring::Event] event details including payload
           def on_worker_process(event)
-            current_span = client.trace('karafka.consumer')
+            current_span = client.trace('karafka.consumer', service: service_name)
             push_tags
 
             job = event[:job]

data/lib/karafka/messages/builders/message.rb

@@ -23,11 +23,15 @@ module Karafka
               received_at: received_at
             ).freeze
 
+            # Get the raw payload
+            payload = kafka_message.payload
+
+            # And nullify it in the kafka message. This can save a lot of memory when used with
+            # the Pro Cleaner API
+            kafka_message.instance_variable_set('@payload', nil)
+
             # Karafka messages cannot be frozen because of the lazy deserialization feature
-            Karafka::Messages::Message.new(
-              kafka_message.payload,
-              metadata
-            )
+            Karafka::Messages::Message.new(payload, metadata)
           end
         end
       end

data/lib/karafka/pro/active_job/consumer.rb

@@ -25,7 +25,7 @@ module Karafka
       class Consumer < ::Karafka::ActiveJob::Consumer
         # Runs ActiveJob jobs processing and handles lrj if needed
         def consume
-          messages.each do |message|
+          messages.each(clean: true) do |message|
             # If for any reason we've lost this partition, not worth iterating over new messages
             # as they are no longer ours
             break if revoked?

data/lib/karafka/pro/cleaner/errors.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner related errors
+      module Errors
+        # Base for all the clearer errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
+        # Raised when trying to deserialize a message that has already been cleaned
+        MessageCleanedError = Class.new(BaseError)
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner/messages/message.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      # Cleaner messages components related enhancements
+      module Messages
+        # Extensions to the message that allow for granular memory control on a per message basis
+        module Message
+          # @return [Object] lazy-deserialized data (deserialized upon first request)
+          def payload
+            # If message has already been cleaned, it cannot be deserialized again
+            cleaned? ? raise(Errors::MessageCleanedError) : super
+          end
+
+          # @return [Boolean] true if the message has been cleaned
+          def cleaned?
+            @raw_payload == false
+          end
+
+          # Cleans the message payload and removes the deserialized data references
+          # This is useful when working with big messages that take a lot of space.
+          #
+          # After the message content is no longer needed, it can be removed so it does not consume
+          # space anymore.
+          def clean!
+            @deserialized = false
+            @raw_payload = false
+            @payload = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner/messages/messages.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    module Cleaner
+      module Messages
+        # Extensions to the messages batch allowing for automatic cleaning of each message after
+        # message is processed.
+        module Messages
+          # @param clean [Boolean] do we want to clean each message after we're done working with
+          #   it.
+          # @yield block we want to execute per each message
+          #
+          # @note Cleaning messages after we're done with each of them and did not fail does not
+          #   affect any other functionalities. The only thing that is crucial is to make sure,
+          #   that if DLQ is used, that we mark each message as consumed when using this API as
+          #   otherwise a cleaned message may be dispatched and that should never happen
+          def each(clean: false)
+            @messages_array.each do |message|
+              yield(message)
+
+              next unless clean
+
+              message.clean!
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/cleaner.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# This Karafka component is a Pro component under a commercial license.
+# This Karafka component is NOT licensed under LGPL.
+#
+# All of the commercial components are present in the lib/karafka/pro directory of this
+# repository and their usage requires commercial license agreement.
+#
+# Karafka has also commercial-friendly license, commercial support and commercial components.
+#
+# By sending a pull request to the pro components, you are agreeing to transfer the copyright of
+# your code to Maciej Mensfeld.
+
+module Karafka
+  module Pro
+    # Feature that introduces a granular memory management for each message and messages iterator
+    #
+    # It allows for better resource allocation by providing an API to clear payload and raw payload
+    # from a message after those are no longer needed but before whole messages are freed and
+    # removed by Ruby GC.
+    #
+    # This can be useful when processing bigger batches or bigger messages one after another and
+    # wanting not to have all of the data loaded into memory.
+    #
+    # Can yield significant memory savings (up to 80%).
+    module Cleaner
+      class << self
+        # @param _config [Karafka::Core::Configurable::Node] root node config
+        def pre_setup(_config)
+          ::Karafka::Messages::Message.prepend(Messages::Message)
+          ::Karafka::Messages::Messages.prepend(Messages::Messages)
+        end
+
+        # @param _config [Karafka::Core::Configurable::Node] root node config
+        def post_setup(_config)
+          true
+        end
+      end
+    end
+  end
+end

data/lib/karafka/pro/encryption/errors.rb

@@ -16,8 +16,11 @@ module Karafka
     module Encryption
       # Encryption related errors
       module Errors
+        # Base for all the encryption errors
+        BaseError = Class.new(::Karafka::Errors::BaseError)
+
         # Raised when we have encountered encryption key with version we do not have
-        PrivateKeyNotFound = Class.new(::Karafka::Errors::BaseError)
+        PrivateKeyNotFound = Class.new(BaseError)
       end
     end
   end

data/lib/karafka/pro/loader.rb

@@ -69,7 +69,8 @@ module Karafka
         # @return [Array<Module>] extra non-routing related pro features
         def features
           [
-            Encryption
+            Encryption,
+            Cleaner
           ]
         end
 

data/lib/karafka/pro/processing/strategies/dlq/default.rb

@@ -28,7 +28,7 @@ module Karafka
 
             # When we encounter non-recoverable message, we skip it and go on with our lives
             def handle_after_consume
-              coordinator.on_finished do
+              coordinator.on_finished do |last_group_message|
                 return if revoked?
 
                 if coordinator.success?
@@ -36,7 +36,7 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  mark_as_consumed(messages.last)
+                  mark_as_consumed(last_group_message)
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries
                   retry_after_pause
                 # If we've reached number of retries that we could, we need to skip the first
@@ -74,6 +74,12 @@ module Karafka
             # @param skippable_message [Array<Karafka::Messages::Message>] message we want to
             #   dispatch to DLQ
             def dispatch_to_dlq(skippable_message)
+              # DLQ should never try to dispatch a message that was cleaned. It message was
+              # cleaned, we will not have all the needed data. If you see this error, it means
+              # that your processing flow is not as expected and you have cleaned message that
+              # should not be cleaned as it should go to the DLQ
+              raise(Cleaner::Errors::MessageCleanedError) if skippable_message.cleaned?
+
               producer.produce_async(
                 build_dlq_message(
                   skippable_message

data/lib/karafka/pro/processing/strategies/dlq/ftr.rb

@@ -31,7 +31,7 @@ module Karafka
             # DLQ flow is standard here, what is not, is the success component where we need to
             # take into consideration the filtering
             def handle_after_consume
-              coordinator.on_finished do
+              coordinator.on_finished do |last_group_message|
                 return if revoked?
 
                 if coordinator.success?
@@ -39,7 +39,7 @@ module Karafka
 
                   return if coordinator.manual_pause?
 
-                  mark_as_consumed(messages.last)
+                  mark_as_consumed(last_group_message)
 
                   handle_post_filtering
                 elsif coordinator.pause_tracker.attempt <= topic.dead_letter_queue.max_retries

data/lib/karafka/version.rb

@@ -3,5 +3,5 @@
 # Main module namespace
 module Karafka
   # Current Karafka version
-  VERSION = '2.1.11'
+  VERSION = '2.1.13'
 end

data.tar.gz.sig

@@ -1,3 +1,6 @@
-g+��\)E"nޗCD�\�S�)�(�U��M��Q
-��*PV=���
&{S?����n��WM����
-�(9H��(�eݜ|5�j�M�y!쁦�w&��\wh? �ʺA[�*����Z�#DY���!1�g��=�W�ʉ����~��7{�
+�ֈ�
+y߀�T�
+:�
+9�Um(��疈tg�y/#�l]D��2���KX��E㜔�R��:���UM�
+�s8׾4*mɦh*�;[�\m>
Q�n��g�T.�x�����Mǎ*O����-�`8���"��+?��M��pԏ�]�� ���d�&�I�Ky��VB��dl��'
+n=��Vp�Z�Β7U�/n���I>���k0��%*�;N�e�����]��[��gtU�4�9f#r,|����6Un��W'����I,��N��:Pf�F�^�㎌r����FR��N����gd2���S�`��Է��V

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka
 version: !ruby/object:Gem::Version
-  version: 2.1.11
+  version: 2.1.13
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   AnG1dJU+yL2BK7vaVytLTstJME5mepSZ46qqIJXMuWob/YPDmVaBF39TDSG9e34s
   msG3BiCqgOgHAnL23+CN3Rt8MsuRfEtoTKpJVcCfoEoNHOkc
   -----END CERTIFICATE-----
-date: 2023-08-23 00:00:00.000000000 Z
+date: 2023-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: karafka-core
@@ -141,6 +141,7 @@ files:
 - bin/create_token
 - bin/integrations
 - bin/karafka
+- bin/record_rss
 - bin/rspecs
 - bin/scenario
 - bin/stress_many
@@ -219,6 +220,10 @@ files:
 - lib/karafka/pro/active_job/consumer.rb
 - lib/karafka/pro/active_job/dispatcher.rb
 - lib/karafka/pro/active_job/job_options_contract.rb
+- lib/karafka/pro/cleaner.rb
+- lib/karafka/pro/cleaner/errors.rb
+- lib/karafka/pro/cleaner/messages/message.rb
+- lib/karafka/pro/cleaner/messages/messages.rb
 - lib/karafka/pro/encryption.rb
 - lib/karafka/pro/encryption/cipher.rb
 - lib/karafka/pro/encryption/contracts/config.rb