karafka 2.5.1 → 2.5.2

data/examples/payloads/json/sample_set_03/event_type_2.json

@@ -0,0 +1,263 @@
+{
+  "connect.name": "production.public.automation_actions.Envelope",
+  "connect.version": 1,
+  "fields": [
+    {
+      "default": null,
+      "name": "before",
+      "type": [
+        "null",
+        {
+          "connect.name": "production.public.automation_actions.Value",
+          "fields": [
+            {
+              "default": "00000000-0000-0000-0000-000000000000",
+              "name": "id",
+              "type": {
+                "connect.default": "00000000-0000-0000-0000-000000000000",
+                "connect.name": "io.debezium.data.Uuid",
+                "connect.version": 1,
+                "type": "string"
+              }
+            },
+            {
+              "name": "automation_id",
+              "type": "int"
+            },
+            {
+              "default": null,
+              "name": "card_id",
+              "type": [
+                "null",
+                "int"
+              ]
+            },
+            {
+              "name": "organization_id",
+              "type": "int"
+            },
+            {
+              "default": null,
+              "name": "user_id",
+              "type": [
+                "null",
+                "int"
+              ]
+            },
+            {
+              "name": "action_name",
+              "type": "string"
+            },
+            {
+              "default": null,
+              "name": "finished_at",
+              "type": [
+                "null",
+                {
+                  "connect.name": "io.debezium.time.MicroTimestamp",
+                  "connect.version": 1,
+                  "type": "long"
+                }
+              ]
+            },
+            {
+              "default": 0,
+              "name": "status",
+              "type": [
+                {
+                  "connect.default": 0,
+                  "type": "int"
+                },
+                "null"
+              ]
+            },
+            {
+              "default": null,
+              "name": "deleted_at",
+              "type": [
+                "null",
+                {
+                  "connect.name": "io.debezium.time.MicroTimestamp",
+                  "connect.version": 1,
+                  "type": "long"
+                }
+              ]
+            },
+            {
+              "name": "created_at",
+              "type": {
+                "connect.name": "io.debezium.time.MicroTimestamp",
+                "connect.version": 1,
+                "type": "long"
+              }
+            },
+            {
+              "name": "updated_at",
+              "type": {
+                "connect.name": "io.debezium.time.MicroTimestamp",
+                "connect.version": 1,
+                "type": "long"
+              }
+            },
+            {
+              "default": null,
+              "name": "response",
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            {
+              "default": null,
+              "name": "executed",
+              "type": [
+                "null",
+                "boolean"
+              ]
+            }
+          ],
+          "name": "Value",
+          "type": "record"
+        }
+      ]
+    },
+    {
+      "default": null,
+      "name": "after",
+      "type": [
+        "null",
+        "Value"
+      ]
+    },
+    {
+      "name": "source",
+      "type": {
+        "connect.name": "io.debezium.connector.v2.postgresql.Source",
+        "fields": [
+          {
+            "name": "version",
+            "type": "string"
+          },
+          {
+            "name": "connector",
+            "type": "string"
+          },
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "ts_ms",
+            "type": "long"
+          },
+          {
+            "default": "false",
+            "name": "snapshot",
+            "type": [
+              {
+                "connect.default": "false",
+                "connect.name": "io.debezium.data.Enum",
+                "connect.parameters": {
+                  "allowed": "true,last,false,incremental"
+                },
+                "connect.version": 1,
+                "type": "string"
+              },
+              "null"
+            ]
+          },
+          {
+            "name": "db",
+            "type": "string"
+          },
+          {
+            "default": null,
+            "name": "sequence",
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          {
+            "name": "schema",
+            "type": "string"
+          },
+          {
+            "name": "table",
+            "type": "string"
+          },
+          {
+            "default": null,
+            "name": "txId",
+            "type": [
+              "null",
+              "long"
+            ]
+          },
+          {
+            "default": null,
+            "name": "lsn",
+            "type": [
+              "null",
+              "long"
+            ]
+          },
+          {
+            "default": null,
+            "name": "xmin",
+            "type": [
+              "null",
+              "long"
+            ]
+          }
+        ],
+        "name": "Source",
+        "namespace": "io.debezium.connector.v2.postgresql",
+        "type": "record"
+      }
+    },
+    {
+      "name": "op",
+      "type": "string"
+    },
+    {
+      "default": null,
+      "name": "ts_ms",
+      "type": [
+        "null",
+        "long"
+      ]
+    },
+    {
+      "default": null,
+      "name": "transaction",
+      "type": [
+        "null",
+        {
+          "connect.name": "event.block",
+          "connect.version": 1,
+          "fields": [
+            {
+              "name": "id",
+              "type": "string"
+            },
+            {
+              "name": "total_order",
+              "type": "long"
+            },
+            {
+              "name": "data_collection_order",
+              "type": "long"
+            }
+          ],
+          "name": "block",
+          "namespace": "event",
+          "type": "record"
+        }
+      ]
+    }
+  ],
+  "name": "Envelope",
+  "namespace": "production.public.automation_actions",
+  "type": "record"
+}

data/examples/payloads/json/sample_set_03/event_type_3.json

@@ -0,0 +1,41 @@
+[
+  {
+    "timestamp": 1111111111111,
+    "timestampType": "TYPE",
+    "partition": 0,
+    "offset": 0,
+    "key": "0000000000000000000000000000000000000000000000000000000000000000",
+    "value": {
+      "id": "0000000000000000000000000000000000000000000000000000000000000000",
+      "type": "event_type",
+      "data": {
+        "user_id": 123456789,
+        "user_email": "user@example.com",
+        "org_id": 987654321,
+        "token_id": 111111111,
+        "token_type": "token_type",
+        "request_id": "request-id",
+        "request_host": "api.example.com",
+        "request_path": "/some/path",
+        "remote_ip": "0.0.0.0",
+        "request_graphql": true,
+        "user_agent": "client-library/x.y.z",
+        "request_method": "METHOD",
+        "graphql_data": {
+          "errored": false,
+          "complexity": 0,
+          "depth": 0,
+          "operation_name": null,
+          "operation_type": "operation",
+          "resources": ["resource"]
+        },
+        "response_code": "200",
+        "type": "custom"
+      },
+      "source": "some-source",
+      "version": 1,
+      "timestamp": "1111111111111"
+    },
+    "headers": []
+  }
+]

data/karafka.gemspec

@@ -7,7 +7,7 @@ require 'karafka/version'
 
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
-  spec.version     = ::Karafka::VERSION
+  spec.version     = Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -67,7 +67,7 @@ module ActiveJob
 
       # @return [Boolean] should we stop the job. Used by the ActiveJob continuation feature
       def stopping?
-        Karafka::App.done?
+        ::Karafka::App.done?
       end
     end
   end

data/lib/karafka/active_job/consumer.rb

@@ -6,6 +6,10 @@ module Karafka
     # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
     # It marks the offset after each message, so we make sure, none of the jobs is executed twice
     class Consumer < ::Karafka::BaseConsumer
+      include Helpers::ConfigImporter.new(
+        deserializer: %i[internal active_job deserializer]
+      )
+
       # Executes the ActiveJob logic
       # @note ActiveJob does not support batches, so we just run one message after another
       def consume
@@ -42,7 +46,7 @@ module Karafka
         # We technically speaking could set this as deserializer and reference it from the
         # message instead of using the `#raw_payload`. This is not done on purpose to simplify
         # the ActiveJob setup here
-        yield ::ActiveSupport::JSON.decode(job_message.raw_payload)
+        yield deserializer.deserialize(job_message)
       end
     end
   end

data/lib/karafka/active_job/current_attributes/job_wrapper.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Simple wrapper that presents a job hash with current attributes injected.
+      #
+      # This wrapper exists to pass a modified job hash to the deserializer without
+      # modifying the original ActiveJob::Base instance. We cannot modify the job instance
+      # directly because:
+      #
+      # 1. Thread safety: Modifying job instances with singleton methods could cause
+      #    concurrency issues in multi-threaded environments
+      # 2. Rails ownership: ActiveJob::Base is a Rails class we don't control, and
+      #    monkey-patching it could break with Rails updates
+      # 3. Side effects: Modifying the job instance could affect other parts of the
+      #    application that use the same job object
+      #
+      # The wrapper implements only the #serialize method that the Deserializer expects,
+      # returning our pre-computed hash with current attributes already injected.
+      #
+      # @example Using JobWrapper with a modified job hash
+      #   job_hash = {
+      #     'job_class' => 'MyJob',
+      #     'arguments' => [1, 2, 3],
+      #     'cattr_0' => { 'user_id' => 123 }
+      #   }
+      #   wrapper = JobWrapper.new(job_hash)
+      #   wrapper.serialize # => returns the job_hash
+      class JobWrapper
+        # @param job_hash [Hash] the job hash with current attributes already injected
+        def initialize(job_hash)
+          @job_hash = job_hash
+        end
+
+        # Returns the job hash with current attributes injected
+        #
+        # @return [Hash] the job hash with current attributes injected
+        def serialize
+          @job_hash
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes/loading.rb

@@ -8,7 +8,7 @@ module Karafka
       module Loading
         # @param job_message [Karafka::Messages::Message] message with active job
         def with_deserialized_job(job_message)
-          super(job_message) do |job|
+          super do |job|
             resetable = []
 
             _cattr_klasses.each do |key, cattr_klass_str|

data/lib/karafka/active_job/current_attributes/persistence.rb

@@ -4,23 +4,35 @@ module Karafka
   module ActiveJob
     module CurrentAttributes
       # Module adding the current attributes persistence into the ActiveJob jobs
+      # This module wraps the Dispatcher#serialize_job to inject current attributes
       module Persistence
-        # Alters the job serialization to inject the current attributes into the json before we
-        # send it to Kafka
+        # Wraps the Dispatcher#serialize_job to inject current attributes before serialization
+        # This allows us to modify the job before it's serialized without modifying ActiveJob::Base
         #
-        # @param job [ActiveJob::Base] job
+        # @param job [ActiveJob::Base] the original job to serialize
+        # @return [String] serialized job payload with current attributes injected
+        #
+        # @note This method creates a JobWrapper internally and passes it to the parent's
+        #   serialize_job method. The wrapper is transparent to the deserializer.
         def serialize_job(job)
-          json = super(job)
+          # Get the job hash
+          job_hash = job.serialize
 
+          # Inject current attributes
           _cattr_klasses.each do |key, cattr_klass_str|
-            next if json.key?(key)
+            next if job_hash.key?(key)
 
             attrs = cattr_klass_str.constantize.attributes
 
-            json[key] = attrs unless attrs.empty?
+            job_hash[key] = attrs unless attrs.empty?
           end
 
-          json
+          # Wrap the modified hash in a simple object that implements #serialize
+          # This avoids modifying the original job instance
+          wrapper = JobWrapper.new(job_hash)
+
+          # Pass the wrapper to the deserializer
+          super(wrapper)
         end
       end
     end

data/lib/karafka/active_job/current_attributes.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'active_support/current_attributes'
+require_relative 'current_attributes/job_wrapper'
 require_relative 'current_attributes/loading'
 require_relative 'current_attributes/persistence'
 

data/lib/karafka/active_job/deserializer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    # Default deserializer for ActiveJob jobs
+    #
+    # @note Despite the name, this class handles both serialization (job to Kafka payload) and
+    #   deserialization (Kafka message to job). It's called "Deserializer" to align with Karafka's
+    #   naming conventions where message consumption is the primary concern.
+    #
+    # This class can be inherited and its methods can be overridden to support
+    # custom payload formats (e.g., Avro, Protobuf, MessagePack)
+    #
+    # @example Wrapping jobs in a custom envelope with metadata
+    #   class EnvelopedJobDeserializer < Karafka::ActiveJob::Deserializer
+    #     def serialize(job)
+    #       # Wrap the job in an envelope with additional metadata
+    #       envelope = {
+    #         version: 1,
+    #         produced_at: Time.now.iso8601,
+    #         producer: 'my-app',
+    #         payload: job.serialize
+    #       }
+    #       ::ActiveSupport::JSON.encode(envelope)
+    #     end
+    #
+    #     def deserialize(message)
+    #       # Extract the job from the envelope
+    #       envelope = ::ActiveSupport::JSON.decode(message.raw_payload)
+    #
+    #       # Could validate envelope version, log metadata, etc.
+    #       raise 'Unsupported version' if envelope['version'] != 1
+    #
+    #       # Return the actual job data
+    #       envelope['payload']
+    #     end
+    #   end
+    #
+    #   # Configure in Karafka
+    #   Karafka::App.config.internal.active_job.deserializer = EnvelopedJobDeserializer.new
+    class Deserializer
+      # Serializes an ActiveJob job into a string payload for Kafka
+      #
+      # @param job [ActiveJob::Base, #serialize] job to serialize. The job must respond to
+      #   #serialize which returns a Hash of job attributes. When CurrentAttributes are used,
+      #   this may be a JobWrapper instance instead of the original ActiveJob::Base.
+      # @return [String] serialized job payload
+      def serialize(job)
+        ::ActiveSupport::JSON.encode(job.serialize)
+      end
+
+      # Deserializes a Kafka message payload into an ActiveJob job hash
+      #
+      # @param message [Karafka::Messages::Message] message containing the job
+      # @return [Hash] deserialized job hash
+      def deserialize(message)
+        ::ActiveSupport::JSON.decode(message.raw_payload)
+      end
+    end
+  end
+end

data/lib/karafka/active_job/dispatcher.rb

@@ -4,6 +4,10 @@ module Karafka
   module ActiveJob
     # Dispatcher that sends the ActiveJob job to a proper topic based on the queue name
     class Dispatcher
+      include Helpers::ConfigImporter.new(
+        deserializer: %i[internal active_job deserializer]
+      )
+
       # Defaults for dispatching
       # The can be updated by using `#karafka_options` on the job
       DEFAULTS = {
@@ -18,7 +22,7 @@ module Karafka
         ::Karafka.producer.public_send(
           fetch_option(job, :dispatch_method, DEFAULTS),
           topic: job.queue_name,
-          payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+          payload: serialize_job(job)
         )
       end
 
@@ -34,7 +38,7 @@ module Karafka
 
           dispatches[d_method] << {
             topic: job.queue_name,
-            payload: ::ActiveSupport::JSON.encode(serialize_job(job))
+            payload: serialize_job(job)
           }
         end
 
@@ -48,14 +52,18 @@ module Karafka
 
       # Raises info, that Karafka backend does not support scheduling jobs if someone wants to
       # schedule jobs in the future. It works for past and present because we want to support
-      # things like continuation and `#retry_on` API with no wait and no jitter
+      # things like continuation and `#retry_on` API with no wait and no jitter.
       #
       # @param job [Object] job we cannot enqueue
       # @param timestamp [Time] time when job should run
       #
-      # @note Karafka Pro supports future jobs
+      # @note Karafka Pro supports future jobs via the Scheduled Messages feature
+      #
+      # @note For ActiveJob Continuation to work without Pro, configure your continuable jobs:
+      #   self.resume_options = { wait: 0 }
       #
-      # @note In order for jobs to work with this you need to set jitter to false and no wait
+      # @note For `#retry_on` to work without Pro, configure with:
+      #   retry_on SomeError, wait: 0, jitter: 0
       def dispatch_at(job, timestamp)
         # Dispatch at is used by some of the ActiveJob features that actually do not back-off
         # but things go via this API nonetheless.
@@ -64,13 +72,31 @@ module Karafka
         else
           raise NotImplementedError, <<~ERROR_MESSAGE
             This queueing backend does not support scheduling future jobs.
-            Consider using Karafka Pro, which supports this via the Scheduled Messages feature.
+
+            If you're using ActiveJob Continuation, configure your jobs with:
+              self.resume_options = { wait: 0 }
+
+            If you're using retry_on, configure with:
+              retry_on SomeError, wait: 0, jitter: 0
+
+            For full support of delayed job execution, consider using Karafka Pro with Scheduled Messages.
           ERROR_MESSAGE
         end
       end
 
       private
 
+      # Serializes a job using the configured deserializer
+      # This method serves as an extension point and can be wrapped by modules like
+      # CurrentAttributes::Persistence
+      #
+      # @param job [ActiveJob::Base, CurrentAttributes::Persistence::JobWrapper] job to serialize.
+      #   When CurrentAttributes are used, this may be a JobWrapper instead of the original job.
+      # @return [String] serialized job payload
+      def serialize_job(job)
+        deserializer.serialize(job)
+      end
+
       # @param job [ActiveJob::Base] job
       # @param key [Symbol] key we want to fetch
       # @param defaults [Hash]
@@ -81,12 +107,6 @@ module Karafka
           .karafka_options
           .fetch(key, defaults.fetch(key))
       end
-
-      # @param job [ActiveJob::Base] job
-      # @return [Hash] json representation of the job
-      def serialize_job(job)
-        job.serialize
-      end
     end
   end
 end

data/lib/karafka/active_job/job_options_contract.rb

@@ -8,10 +8,8 @@ module Karafka
     #   all in the same place
     class JobOptionsContract < Contracts::Base
       configure do |config|
-        config.error_messages = YAML.safe_load(
-          File.read(
-            File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
-          )
+        config.error_messages = YAML.safe_load_file(
+          File.join(Karafka.gem_root, 'config', 'locales', 'errors.yml')
         ).fetch('en').fetch('validations').fetch('job_options')
       end
 

data/lib/karafka/admin/acl.rb

@@ -98,8 +98,10 @@ module Karafka
         PERMISSION_TYPES_MAP
       ].freeze
 
-      private_constant :RESOURCE_TYPES_MAP, :RESOURCE_PATTERNS_TYPE_MAP, :OPERATIONS_MAP,
-                       :PERMISSION_TYPES_MAP, :ALL_MAPS
+      private_constant(
+        :RESOURCE_TYPES_MAP, :RESOURCE_PATTERNS_TYPE_MAP, :OPERATIONS_MAP, :PERMISSION_TYPES_MAP,
+        :ALL_MAPS
+      )
 
       # Class level APIs that operate on Acl instances and/or return Acl instances.
       # @note For the sake of consistency all methods from this API return array of Acls
@@ -187,8 +189,10 @@ module Karafka
         end
       end
 
-      attr_reader :resource_type, :resource_name, :resource_pattern_type, :principal, :host,
-                  :operation, :permission_type
+      attr_reader(
+        :resource_type, :resource_name, :resource_pattern_type, :principal, :host, :operation,
+        :permission_type
+      )
 
       # Initializes a new Acl instance with specified attributes.
       #

data/lib/karafka/admin/configs/config.rb

@@ -71,10 +71,12 @@ module Karafka
         def synonym? = @synonym.positive?
 
         # @return [Hash] hash that we can use to operate with rdkafka
-        def to_native_hash = {
-          name: name,
-          value: value
-        }.freeze
+        def to_native_hash
+          {
+            name: name,
+            value: value
+          }.freeze
+        end
       end
     end
   end