karafka-core 2.0.4 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb55ae933175f1fc17993fb6bb1e980a7a3494814ceb6ff0b88e3b5b069af2f5
-  data.tar.gz: af485691da69a93ad270db1d864aac83abe5f84de6e234a51888c7a6488503bb
+  metadata.gz: 9e11bdb1c5a745e22d10b63dff4b2fb0a7a30dd1812aa8f7d3ad555d061dd451
+  data.tar.gz: 27d163cb019a0b57109bb4e6d53f3adce889a1d315f708f4100ffcee4d4885f8
 SHA512:
-  metadata.gz: 2c6504961821ce9d3cc68d552a1b010a4c102a3d17570d6f950b541ebf48740df66a50b55ff2cfd1f086ebbf62f2bd0e7b3742a37ca1648c9acacfae0a439192
-  data.tar.gz: 641d243fbcf7ae79a34dcbfd1f1e66499d9bf3db4b8e6ec4a401844b0a4bc95e10a82fba45e2f94b39d082c29f17e9f786b1f6b4b4af1781ea01d5264ae949dc
+  metadata.gz: d838c690a75dd37b1a15b666df8eafbe38c5f8cd95a38db9d7350a0753220b1c168d9ab153284b2dc5947d920ea26b0056871879cb2ca8bc8fba65f09920304b
+  data.tar.gz: ec6a1bd9ba1734975d8bafb0d57009e87101de8821caf96d30aa14e358169c0042f9744e7b1db951da959d7bcae19e4a202ad6c2f997bf5e40d6a7cceb1f75c1

data/.ruby-version

@@ -1 +1 @@
-3.1.2
+3.1.3

data/CHANGELOG.md

@@ -1,6 +1,11 @@
 # Karafka core changelog
 
-# 2.0.4 (2022-11-20)
+## 2.0.5 (2022-12-07)
+- Move `librdkafka` generic (producer and consumer) patches from WaterDrop here.
+- Move dependency on `librdkafka` here from both Karafka and WaterDrop to unify management.
+- Move `CallbacksManager` from WaterDrop because it's shared.
+
+## 2.0.4 (2022-11-20)
 - Disallow publishing events that were not registered.
 - Fix a potential race condition when adding listeners concurrently from multiple threads.
 

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-core (2.0.4)
+    karafka-core (2.0.5)
       concurrent-ruby (>= 1.1)
+      rdkafka (>= 0.12)
 
 GEM
   remote: https://rubygems.org/
@@ -18,9 +19,16 @@ GEM
     docile (1.4.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
+    ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
+    mini_portile2 (2.8.0)
     minitest (5.16.3)
+    rake (13.0.6)
+    rdkafka (0.12.0)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.6)
+      rake (> 12)
     rspec (3.12.0)
       rspec-core (~> 3.12.0)
       rspec-expectations (~> 3.12.0)
@@ -55,4 +63,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.24
+   2.3.26

data/karafka-core.gemspec

@@ -15,7 +15,9 @@ Gem::Specification.new do |spec|
   spec.summary     = 'Karafka ecosystem core modules'
   spec.description = 'A toolset of small support modules used throughout the Karafka ecosystem'
   spec.licenses    = %w[MIT]
+
   spec.add_dependency 'concurrent-ruby', '>= 1.1'
+  spec.add_dependency 'rdkafka', '>= 0.12'
 
   spec.required_ruby_version = '>= 2.6.0'
 

data/lib/karafka/core/configurable/node.rb

@@ -21,7 +21,6 @@ module Karafka
           @name = name
           @children = []
           @nestings = nestings
-          @compiled = false
           instance_eval(&nestings)
         end
 
@@ -44,7 +43,7 @@ module Karafka
         # Compile settings, allow for overrides via yielding
         # @return [Node] returns self after configuration
         def configure
-          compile unless @compiled
+          compile
           yield(self) if block_given?
           self
         end
@@ -98,8 +97,6 @@ module Karafka
 
             public_send("#{value.name}=", initialized)
           end
-
-          @compiled = true
         end
       end
     end

data/lib/karafka/core/instrumentation/callbacks_manager.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Karafka instrumentation related shared components
+    module Instrumentation
+      # This manager allows us to register multiple callbacks into a hook that is suppose to support
+      # a single callback
+      class CallbacksManager
+        # @return [::Karafka::Core::Instrumentation::CallbacksManager]
+        def initialize
+          @callbacks = Concurrent::Hash.new
+        end
+
+        # Invokes all the callbacks registered one after another
+        #
+        # @param args [Object] any args that should go to the callbacks
+        # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
+        #   callbacks and add new at the same time. Since we don't know when and in what thread
+        #   things are going to be added to the manager, we need to extract values into an array and
+        #   run it. That way we can add new things the same time.
+        def call(*args)
+          @callbacks.values.each { |callback| callback.call(*args) }
+        end
+
+        # Adds a callback to the manager
+        #
+        # @param id [String] id of the callback (used when deleting it)
+        # @param callable [#call] object that responds to a `#call` method
+        def add(id, callable)
+          @callbacks[id] = callable
+        end
+
+        # Removes the callback from the manager
+        # @param id [String] id of the callback we want to remove
+        def delete(id)
+          @callbacks.delete(id)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/instrumentation.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # All the instrumentation shared across Karafka ecosystem
+    module Instrumentation
+      class << self
+        # Builds a manager for statistics callbacks
+        # @return [WaterDrop::CallbacksManager]
+        def statistics_callbacks
+          @statistics_callbacks ||= CallbacksManager.new
+        end
+
+        # Builds a manager for error callbacks
+        # @return [WaterDrop::CallbacksManager]
+        def error_callbacks
+          @error_callbacks ||= CallbacksManager.new
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/patches/rdkafka/bindings.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Patches to dependencies and components
+    module Patches
+      # Patches to rdkafka
+      module Rdkafka
+        # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
+        # to work with in a bit different way than rdkafka itself
+        module Bindings
+          class << self
+            # Add extra methods that we need
+            # @param mod [::Rdkafka::Bindings] rdkafka bindings module
+            def included(mod)
+              mod.attach_function :rd_kafka_name, [:pointer], :string
+
+              # Default rdkafka setup for errors doest not propagate client details, thus it always
+              # publishes all the stuff for all rdkafka instances. We change that by providing
+              # function that fetches the instance name, allowing us to have better notifications
+              mod.send(:remove_const, :ErrorCallback)
+              mod.const_set(:ErrorCallback, build_error_callback)
+            end
+
+            # @return [FFI::Function] overwritten callback function
+            def build_error_callback
+              FFI::Function.new(
+                :void, %i[pointer int string pointer]
+              ) do |client_prr, err_code, reason, _opaque|
+                return nil unless ::Rdkafka::Config.error_callback
+
+                name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
+
+                error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+
+                ::Rdkafka::Config.error_callback.call(name, error)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/version.rb

@@ -4,6 +4,6 @@ module Karafka
   module Core
     # Current Karafka::Core version
     # We follow the versioning schema of given Karafka version
-    VERSION = '2.0.4'
+    VERSION = '2.0.5'
   end
 end

data/lib/karafka-core.rb

@@ -2,24 +2,44 @@
 
 %w[
   yaml
+  rdkafka
+
   concurrent/map
   concurrent/hash
   concurrent/array
+
   karafka/core
   karafka/core/version
+
   karafka/core/monitoring
   karafka/core/monitoring/event
   karafka/core/monitoring/monitor
   karafka/core/monitoring/notifications
   karafka/core/monitoring/statistics_decorator
+
   karafka/core/configurable
   karafka/core/configurable/leaf
   karafka/core/configurable/node
+
   karafka/core/contractable/contract
   karafka/core/contractable/result
   karafka/core/contractable/rule
+
+  karafka/core/instrumentation
+  karafka/core/instrumentation/callbacks_manager
+
+  karafka/core/patches/rdkafka/bindings
 ].each { |dependency| require dependency }
 
 # Karafka framework main namespace
 module Karafka
 end
+
+# Patch rdkafka
+::Rdkafka::Bindings.include(::Karafka::Core::Patches::Rdkafka::Bindings)
+
+# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
+# each callback type. Callback manager allows us to register more than one callback
+# @note Those managers are also used by Karafka for consumer related statistics
+::Rdkafka::Config.statistics_callback = ::Karafka::Core::Instrumentation.statistics_callbacks
+::Rdkafka::Config.error_callback = ::Karafka::Core::Instrumentation.error_callbacks

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-core
 version: !ruby/object:Gem::Version
-  version: 2.0.4
+  version: 2.0.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-11-20 00:00:00.000000000 Z
+date: 2022-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -51,6 +51,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rdkafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
 description: A toolset of small support modules used throughout the Karafka ecosystem
 email:
 - contact@karafka.io
@@ -87,11 +101,14 @@ files:
 - lib/karafka/core/contractable/contract.rb
 - lib/karafka/core/contractable/result.rb
 - lib/karafka/core/contractable/rule.rb
+- lib/karafka/core/instrumentation.rb
+- lib/karafka/core/instrumentation/callbacks_manager.rb
 - lib/karafka/core/monitoring.rb
 - lib/karafka/core/monitoring/event.rb
 - lib/karafka/core/monitoring/monitor.rb
 - lib/karafka/core/monitoring/notifications.rb
 - lib/karafka/core/monitoring/statistics_decorator.rb
+- lib/karafka/core/patches/rdkafka/bindings.rb
 - lib/karafka/core/version.rb
 - log/.gitkeep
 homepage: https://karafka.io
@@ -120,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Karafka ecosystem core modules