karafka 2.5.1 → 2.5.3

data/examples/payloads/json/sample_set_02/download.json

@@ -0,0 +1,191 @@
+{
+  "audience_data_versions": {
+    "s1_build_epoch": 1634594595,
+    "s2_build_epoch": 1634093288,
+    "s3_build_epoch": 1634095356
+  },
+  "combination_cache_hits": null,
+  "content_insertion_info": {
+    "ad_rep_account_revenue": {
+      "code": "USD",
+      "scale": "Nano",
+      "value": 0
+    },
+    "ad_rep_account": {
+      "id": "f8a3c7d2-9b41-4e56-a2f3-8d7c1b9e4a6f",
+      "name": "AdOps_Beta"
+    },
+    "advertisement": {
+      "duration": {
+        "scale": "Micro",
+        "value": 29022040
+      },
+      "external_identifier": null,
+      "id": "a9e2f4b7-3c81-4d92-b5e6-7a4f8c2d1e9b",
+      "title": "soda",
+      "type": "TraditionalAd"
+    },
+    "platform_revenue": {
+      "code": "USD",
+      "scale": "Nano",
+      "value": 0
+    },
+    "audience": null,
+    "brand": {
+      "id": "d4c8e1a9-7f62-4b3d-9e85-c2a7f4b9d1e6",
+      "name": "Brand_Beta"
+    },
+    "campaign": {
+      "id": "b7d3a5e8-4c91-4f2b-a6d9-1e8c7f3a5b4d",
+      "title": "Pepsi Max",
+      "frequency_counting_targets": null
+    },
+    "campaign_value": {
+      "code": "USD",
+      "scale": "Nano",
+      "value": 500000
+    },
+    "normalized_campaign_value": {
+      "code": "USD",
+      "scale": "Nano",
+      "value": 500000
+    },
+    "external_payload_id": "c5e9a2f7-8d41-4b36-9e72-a1f4c8b3d7e5",
+    "marker_point": {
+      "id": "e2f7c9a4-1d58-4b63-8e49-f7a3c2d8b1e6",
+      "episode_version_id": "a8c4e7f2-9b31-4d65-b8a9-e3f7c1d4a2b6",
+      "maximum_content_count": 1,
+      "maximum_content_duration": {
+        "scale": "Micro",
+        "value": 120000000
+      },
+      "position_type": "Preroll",
+      "position_sequence": 1
+    },
+    "position_sequence": 1,
+    "series_revenue": {
+      "code": "USD",
+      "scale": "Nano",
+      "value": 0
+    }
+  },
+  "content_insertion_system_type": "ServerSide",
+  "count": 1,
+  "deduplication_keys": null,
+  "distribution_characteristic_id": "c9e3f2a7-4d81-4b56-9e73-a8f2c4d1b7e6",
+  "distribution_identification_method_id": "f7a2c8e4-9b31-4d65-8e49-b3f7a1c2d9e8",
+  "episode": {
+    "created_at": 1698698168000,
+    "capacities": [
+      {
+        "position_type": "Preroll",
+        "count": 1,
+        "duration": 120
+      },
+      {
+        "position_type": "Midroll",
+        "count": 0,
+        "duration": 0
+      },
+      {
+        "position_type": "Postroll",
+        "count": 0,
+        "duration": 0
+      }
+    ],
+    "duration": {
+      "scale": "Micro",
+      "value": 4204747760
+    },
+    "episode_number": null,
+    "id": "d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6",
+    "title": "Mountain Breeze",
+    "episode_version_id": "a8c4e7f2-9b31-4d65-b8a9-e3f7c1d4a2b6"
+  },
+  "feed": {
+    "id": "b4f9c7e2-8a31-4d56-9e82-a7f3c1d4b8e9",
+    "type": null
+  },
+  "feed_item": {
+    "id": "d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6"
+  },
+  "id": "e7c2f9a4-8d31-4b65-9e78-a3f4c7d2b1e9",
+  "listened_at": 1747928515000000,
+  "listened_at_ms": 1747928515000,
+  "listener": {
+    "connection_type": "Cellular",
+    "ip_address": "172.58.243.167",
+    "ip_address_encryption_format": "None",
+    "isp": {
+      "as_organization": "WIRELESS-MOBILE",
+      "asn": 8445,
+      "name": "Random Telecom",
+      "organization": "Random Telecom"
+    },
+    "location": {
+      "continent": {
+        "code": "NA",
+        "geoname_id": 6255149,
+        "name": "North America"
+      },
+      "country": {
+        "code": "US",
+        "geoname_id": 6252001
+      },
+      "designated_market_area": {
+        "code": "524",
+        "name": "Atlanta, GA"
+      },
+      "is_in_eu": null,
+      "locality": {
+        "geoname_id": 4180439,
+        "name": "Atlanta"
+      },
+      "region": {
+        "geoname_id": 4197000,
+        "name": "Georgia"
+      },
+      "lat": 33.748997,
+      "lng": -84.387985,
+      "postal_code": "20309"
+    },
+    "rss_browser": null,
+    "time_zone": "America/New_York",
+    "user_agent": {
+      "browser": "Safari",
+      "mobile": true,
+      "operating_system": "iOS",
+      "platform": null,
+      "user_agent": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
+      "user_agent_hash": "9k742p/X5m"
+    }
+  },
+  "network": null,
+  "playback_group_id": "f9c3e7a2-8d41-4b56-9e73-a8f2c4d1b7e6",
+  "request_duration": 0,
+  "request_headers": {
+    "SERVER_PORT": "80",
+    "SERVER_PROTOCOL": "HTTP/1.1",
+    "REMOTE_ADDR": "172.58.243.167",
+    "ORIGINAL_SCRIPT_NAME": "",
+    "HTTP_VERSION": "HTTP/1.1",
+    "HTTP_HOST": "foo.bar.com",
+    "REQUEST_METHOD": "GET",
+    "HTTP_ACCEPT": "*/*",
+    "SCRIPT_NAME": "",
+    "HTTP_USER_AGENT": "Mozilla/5.0 (iPhone; CPU iPhone OS 16_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.5 Mobile/15E148 Safari/604.1",
+    "REQUEST_PATH": "/paths/d8e2a7f4-9c51-4b36-a8e2-f7c4d1b9a3e6",
+    "HTTP_CONNECTION": "close"
+  },
+  "rejection_status": "NotRejected",
+  "root_network": null,
+  "season": null,
+  "series": {
+    "id": "c8e4a7f2-9b31-4d65-8e49-b3f7a1c2d9e8",
+    "title": "Podcast"
+  },
+  "session_id": "a7f2c9e4-8d31-4b56-9e73-a8f4c2d1b7e6",
+  "statistics_system": "TwentyFourHourWindows",
+  "tracking_id": "b9e3f7a2-4c81-4d56-9e78-a3f4c7d2b1e9",
+  "tracking_id_source": "UniqueClientAttributes"
+}

data/examples/payloads/json/sample_set_03/event_type_1.json

@@ -0,0 +1,18 @@
+{
+  "connect.name": "production.public.automation_actions.Key",
+  "fields": [
+    {
+      "default": "00000000-0000-0000-0000-000000000000",
+      "name": "id",
+      "type": {
+        "connect.default": "00000000-0000-0000-0000-000000000000",
+        "connect.name": "io.debezium.data.Uuid",
+        "connect.version": 1,
+        "type": "string"
+      }
+    }
+  ],
+  "name": "Key",
+  "namespace": "production.public.automation_actions",
+  "type": "record"
+}

data/examples/payloads/json/sample_set_03/event_type_2.json

@@ -0,0 +1,263 @@
+{
+  "connect.name": "production.public.automation_actions.Envelope",
+  "connect.version": 1,
+  "fields": [
+    {
+      "default": null,
+      "name": "before",
+      "type": [
+        "null",
+        {
+          "connect.name": "production.public.automation_actions.Value",
+          "fields": [
+            {
+              "default": "00000000-0000-0000-0000-000000000000",
+              "name": "id",
+              "type": {
+                "connect.default": "00000000-0000-0000-0000-000000000000",
+                "connect.name": "io.debezium.data.Uuid",
+                "connect.version": 1,
+                "type": "string"
+              }
+            },
+            {
+              "name": "automation_id",
+              "type": "int"
+            },
+            {
+              "default": null,
+              "name": "card_id",
+              "type": [
+                "null",
+                "int"
+              ]
+            },
+            {
+              "name": "organization_id",
+              "type": "int"
+            },
+            {
+              "default": null,
+              "name": "user_id",
+              "type": [
+                "null",
+                "int"
+              ]
+            },
+            {
+              "name": "action_name",
+              "type": "string"
+            },
+            {
+              "default": null,
+              "name": "finished_at",
+              "type": [
+                "null",
+                {
+                  "connect.name": "io.debezium.time.MicroTimestamp",
+                  "connect.version": 1,
+                  "type": "long"
+                }
+              ]
+            },
+            {
+              "default": 0,
+              "name": "status",
+              "type": [
+                {
+                  "connect.default": 0,
+                  "type": "int"
+                },
+                "null"
+              ]
+            },
+            {
+              "default": null,
+              "name": "deleted_at",
+              "type": [
+                "null",
+                {
+                  "connect.name": "io.debezium.time.MicroTimestamp",
+                  "connect.version": 1,
+                  "type": "long"
+                }
+              ]
+            },
+            {
+              "name": "created_at",
+              "type": {
+                "connect.name": "io.debezium.time.MicroTimestamp",
+                "connect.version": 1,
+                "type": "long"
+              }
+            },
+            {
+              "name": "updated_at",
+              "type": {
+                "connect.name": "io.debezium.time.MicroTimestamp",
+                "connect.version": 1,
+                "type": "long"
+              }
+            },
+            {
+              "default": null,
+              "name": "response",
+              "type": [
+                "null",
+                "string"
+              ]
+            },
+            {
+              "default": null,
+              "name": "executed",
+              "type": [
+                "null",
+                "boolean"
+              ]
+            }
+          ],
+          "name": "Value",
+          "type": "record"
+        }
+      ]
+    },
+    {
+      "default": null,
+      "name": "after",
+      "type": [
+        "null",
+        "Value"
+      ]
+    },
+    {
+      "name": "source",
+      "type": {
+        "connect.name": "io.debezium.connector.v2.postgresql.Source",
+        "fields": [
+          {
+            "name": "version",
+            "type": "string"
+          },
+          {
+            "name": "connector",
+            "type": "string"
+          },
+          {
+            "name": "name",
+            "type": "string"
+          },
+          {
+            "name": "ts_ms",
+            "type": "long"
+          },
+          {
+            "default": "false",
+            "name": "snapshot",
+            "type": [
+              {
+                "connect.default": "false",
+                "connect.name": "io.debezium.data.Enum",
+                "connect.parameters": {
+                  "allowed": "true,last,false,incremental"
+                },
+                "connect.version": 1,
+                "type": "string"
+              },
+              "null"
+            ]
+          },
+          {
+            "name": "db",
+            "type": "string"
+          },
+          {
+            "default": null,
+            "name": "sequence",
+            "type": [
+              "null",
+              "string"
+            ]
+          },
+          {
+            "name": "schema",
+            "type": "string"
+          },
+          {
+            "name": "table",
+            "type": "string"
+          },
+          {
+            "default": null,
+            "name": "txId",
+            "type": [
+              "null",
+              "long"
+            ]
+          },
+          {
+            "default": null,
+            "name": "lsn",
+            "type": [
+              "null",
+              "long"
+            ]
+          },
+          {
+            "default": null,
+            "name": "xmin",
+            "type": [
+              "null",
+              "long"
+            ]
+          }
+        ],
+        "name": "Source",
+        "namespace": "io.debezium.connector.v2.postgresql",
+        "type": "record"
+      }
+    },
+    {
+      "name": "op",
+      "type": "string"
+    },
+    {
+      "default": null,
+      "name": "ts_ms",
+      "type": [
+        "null",
+        "long"
+      ]
+    },
+    {
+      "default": null,
+      "name": "transaction",
+      "type": [
+        "null",
+        {
+          "connect.name": "event.block",
+          "connect.version": 1,
+          "fields": [
+            {
+              "name": "id",
+              "type": "string"
+            },
+            {
+              "name": "total_order",
+              "type": "long"
+            },
+            {
+              "name": "data_collection_order",
+              "type": "long"
+            }
+          ],
+          "name": "block",
+          "namespace": "event",
+          "type": "record"
+        }
+      ]
+    }
+  ],
+  "name": "Envelope",
+  "namespace": "production.public.automation_actions",
+  "type": "record"
+}

data/examples/payloads/json/sample_set_03/event_type_3.json

@@ -0,0 +1,41 @@
+[
+  {
+    "timestamp": 1111111111111,
+    "timestampType": "TYPE",
+    "partition": 0,
+    "offset": 0,
+    "key": "0000000000000000000000000000000000000000000000000000000000000000",
+    "value": {
+      "id": "0000000000000000000000000000000000000000000000000000000000000000",
+      "type": "event_type",
+      "data": {
+        "user_id": 123456789,
+        "user_email": "user@example.com",
+        "org_id": 987654321,
+        "token_id": 111111111,
+        "token_type": "token_type",
+        "request_id": "request-id",
+        "request_host": "api.example.com",
+        "request_path": "/some/path",
+        "remote_ip": "0.0.0.0",
+        "request_graphql": true,
+        "user_agent": "client-library/x.y.z",
+        "request_method": "METHOD",
+        "graphql_data": {
+          "errored": false,
+          "complexity": 0,
+          "depth": 0,
+          "operation_name": null,
+          "operation_type": "operation",
+          "resources": ["resource"]
+        },
+        "response_code": "200",
+        "type": "custom"
+      },
+      "source": "some-source",
+      "version": 1,
+      "timestamp": "1111111111111"
+    },
+    "headers": []
+  }
+]

data/karafka.gemspec

@@ -7,7 +7,7 @@ require 'karafka/version'
 
 Gem::Specification.new do |spec|
   spec.name        = 'karafka'
-  spec.version     = ::Karafka::VERSION
+  spec.version     = Karafka::VERSION
   spec.platform    = Gem::Platform::RUBY
   spec.authors     = ['Maciej Mensfeld']
   spec.email       = %w[contact@karafka.io]
@@ -23,8 +23,8 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency 'base64', '~> 0.2'
   spec.add_dependency 'karafka-core', '>= 2.5.6', '< 2.6.0'
-  spec.add_dependency 'karafka-rdkafka', '>= 0.22.0'
-  spec.add_dependency 'waterdrop', '>= 2.8.9', '< 3.0.0'
+  spec.add_dependency 'karafka-rdkafka', '>= 0.23.1'
+  spec.add_dependency 'waterdrop', '>= 2.8.14', '< 3.0.0'
   spec.add_dependency 'zeitwerk', '~> 2.3'
 
   spec.required_ruby_version = '>= 3.2.0'

data/lib/active_job/queue_adapters/karafka_adapter.rb

@@ -21,7 +21,7 @@ module ActiveJob
     # - No Rails: Inherit from Object (standalone ActiveJob usage)
     #
     # @see https://github.com/sidekiq/sidekiq/issues/6746 Similar issue in Sidekiq
-    base = if defined?(Rails) && defined?(Rails::VERSION)
+    base = if defined?(Rails::VERSION)
              (Rails::VERSION::MAJOR == 7 && Rails::VERSION::MINOR < 2 ? Object : AbstractAdapter)
            else
              # Fallback when Rails is not loaded
@@ -31,7 +31,7 @@ module ActiveJob
     # Karafka adapter for enqueuing jobs
     # This is here for ease of integration with ActiveJob.
     class KarafkaAdapter < base
-      include Karafka::Helpers::ConfigImporter.new(
+      include ::Karafka::Helpers::ConfigImporter.new(
         dispatcher: %i[internal active_job dispatcher]
       )
 
@@ -67,7 +67,7 @@ module ActiveJob
 
       # @return [Boolean] should we stop the job. Used by the ActiveJob continuation feature
       def stopping?
-        Karafka::App.done?
+        ::Karafka::App.done?
       end
     end
   end

data/lib/karafka/active_job/consumer.rb

@@ -5,12 +5,16 @@ module Karafka
   module ActiveJob
     # This is the consumer for ActiveJob that eats the messages enqueued with it one after another.
     # It marks the offset after each message, so we make sure, none of the jobs is executed twice
-    class Consumer < ::Karafka::BaseConsumer
+    class Consumer < Karafka::BaseConsumer
+      include Helpers::ConfigImporter.new(
+        deserializer: %i[internal active_job deserializer]
+      )
+
       # Executes the ActiveJob logic
       # @note ActiveJob does not support batches, so we just run one message after another
       def consume
         messages.each do |message|
-          break if Karafka::App.stopping?
+          break if ::Karafka::App.stopping?
 
           consume_job(message)
 
@@ -42,7 +46,7 @@ module Karafka
         # We technically speaking could set this as deserializer and reference it from the
         # message instead of using the `#raw_payload`. This is not done on purpose to simplify
         # the ActiveJob setup here
-        yield ::ActiveSupport::JSON.decode(job_message.raw_payload)
+        yield deserializer.deserialize(job_message)
       end
     end
   end

data/lib/karafka/active_job/current_attributes/job_wrapper.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Karafka
+  module ActiveJob
+    module CurrentAttributes
+      # Simple wrapper that presents a job hash with current attributes injected.
+      #
+      # This wrapper exists to pass a modified job hash to the deserializer without
+      # modifying the original ActiveJob::Base instance. We cannot modify the job instance
+      # directly because:
+      #
+      # 1. Thread safety: Modifying job instances with singleton methods could cause
+      #    concurrency issues in multi-threaded environments
+      # 2. Rails ownership: ActiveJob::Base is a Rails class we don't control, and
+      #    monkey-patching it could break with Rails updates
+      # 3. Side effects: Modifying the job instance could affect other parts of the
+      #    application that use the same job object
+      #
+      # The wrapper implements only the #serialize method that the Deserializer expects,
+      # returning our pre-computed hash with current attributes already injected.
+      #
+      # @example Using JobWrapper with a modified job hash
+      #   job_hash = {
+      #     'job_class' => 'MyJob',
+      #     'arguments' => [1, 2, 3],
+      #     'cattr_0' => { 'user_id' => 123 }
+      #   }
+      #   wrapper = JobWrapper.new(job_hash)
+      #   wrapper.serialize # => returns the job_hash
+      class JobWrapper
+        # @param job_hash [Hash] the job hash with current attributes already injected
+        def initialize(job_hash)
+          @job_hash = job_hash
+        end
+
+        # Returns the job hash with current attributes injected
+        #
+        # @return [Hash] the job hash with current attributes injected
+        def serialize
+          @job_hash
+        end
+      end
+    end
+  end
+end

data/lib/karafka/active_job/current_attributes/loading.rb

@@ -8,7 +8,7 @@ module Karafka
       module Loading
         # @param job_message [Karafka::Messages::Message] message with active job
         def with_deserialized_job(job_message)
-          super(job_message) do |job|
+          super do |job|
             resetable = []
 
             _cattr_klasses.each do |key, cattr_klass_str|

data/lib/karafka/active_job/current_attributes/persistence.rb

@@ -4,23 +4,35 @@ module Karafka
   module ActiveJob
     module CurrentAttributes
       # Module adding the current attributes persistence into the ActiveJob jobs
+      # This module wraps the Dispatcher#serialize_job to inject current attributes
       module Persistence
-        # Alters the job serialization to inject the current attributes into the json before we
-        # send it to Kafka
+        # Wraps the Dispatcher#serialize_job to inject current attributes before serialization
+        # This allows us to modify the job before it's serialized without modifying ActiveJob::Base
         #
-        # @param job [ActiveJob::Base] job
+        # @param job [ActiveJob::Base] the original job to serialize
+        # @return [String] serialized job payload with current attributes injected
+        #
+        # @note This method creates a JobWrapper internally and passes it to the parent's
+        #   serialize_job method. The wrapper is transparent to the deserializer.
         def serialize_job(job)
-          json = super(job)
+          # Get the job hash
+          job_hash = job.serialize
 
+          # Inject current attributes
           _cattr_klasses.each do |key, cattr_klass_str|
-            next if json.key?(key)
+            next if job_hash.key?(key)
 
             attrs = cattr_klass_str.constantize.attributes
 
-            json[key] = attrs unless attrs.empty?
+            job_hash[key] = attrs unless attrs.empty?
           end
 
-          json
+          # Wrap the modified hash in a simple object that implements #serialize
+          # This avoids modifying the original job instance
+          wrapper = JobWrapper.new(job_hash)
+
+          # Pass the wrapper to the deserializer
+          super(wrapper)
         end
       end
     end