karafka-core 2.0.3 → 2.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a51b03ca1db01c6913a449e8d24f0f6c46a29ceddd262b33999a368425bc44b
-  data.tar.gz: d56de0a9d7f23ffbf1559659ff98fe45053ffeee3b0204513fc56c07245fe0c6
+  metadata.gz: 9e11bdb1c5a745e22d10b63dff4b2fb0a7a30dd1812aa8f7d3ad555d061dd451
+  data.tar.gz: 27d163cb019a0b57109bb4e6d53f3adce889a1d315f708f4100ffcee4d4885f8
 SHA512:
-  metadata.gz: 3d8b26837986d4437489576cfb779d4c27c7db3a2d4930849686bcf252964d2a2f02614591120dd3446950aa80f57a7e5464ee68321b13805aaa7a534a16156e
-  data.tar.gz: 9986d8319b6e881fce6bf91aee9ec086a5b0b57baf8855e52e3dc5db49141212b5e6c71bae394d87e85d6d384b660cd1a082cc73160282b132d89ad8f572c24c
+  metadata.gz: d838c690a75dd37b1a15b666df8eafbe38c5f8cd95a38db9d7350a0753220b1c168d9ab153284b2dc5947d920ea26b0056871879cb2ca8bc8fba65f09920304b
+  data.tar.gz: ec6a1bd9ba1734975d8bafb0d57009e87101de8821caf96d30aa14e358169c0042f9744e7b1db951da959d7bcae19e4a202ad6c2f997bf5e40d6a7cceb1f75c1

data/.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: ['https://karafka.io/#become-pro']

data/.github/workflows/ci.yml

@@ -23,7 +23,7 @@ jobs:
           - ruby: '3.1'
             coverage: 'true'
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: Install package dependencies
         run: "[ -e $APT_DEPS ] || sudo apt-get install -y --no-install-recommends $APT_DEPS"
       - name: Set up Ruby
@@ -48,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Set up Ruby
@@ -67,7 +67,7 @@ jobs:
     strategy:
       fail-fast: false
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
       - name: Run Coditsu

data/.ruby-version

@@ -1 +1 @@
-3.1.2
+3.1.3

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Karafka core changelog
 
+## 2.0.5 (2022-12-07)
+- Move `librdkafka` generic (producer and consumer) patches from WaterDrop here.
+- Move dependency on `librdkafka` here from both Karafka and WaterDrop to unify management.
+- Move `CallbacksManager` from WaterDrop because it's shared.
+
+## 2.0.4 (2022-11-20)
+- Disallow publishing events that were not registered.
+- Fix a potential race condition when adding listeners concurrently from multiple threads.
+
 ## 2.0.3 (2022-10-13)
 - Maintenance release. Cert chain update. No code changes.
 

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    karafka-core (2.0.3)
+    karafka-core (2.0.5)
       concurrent-ruby (>= 1.1)
+      rdkafka (>= 0.12)
 
 GEM
   remote: https://rubygems.org/
@@ -18,22 +19,29 @@ GEM
     docile (1.4.0)
     factory_bot (6.2.1)
       activesupport (>= 5.0.0)
+    ffi (1.15.5)
     i18n (1.12.0)
       concurrent-ruby (~> 1.0)
+    mini_portile2 (2.8.0)
     minitest (5.16.3)
-    rspec (3.11.0)
-      rspec-core (~> 3.11.0)
-      rspec-expectations (~> 3.11.0)
-      rspec-mocks (~> 3.11.0)
-    rspec-core (3.11.0)
-      rspec-support (~> 3.11.0)
-    rspec-expectations (3.11.1)
+    rake (13.0.6)
+    rdkafka (0.12.0)
+      ffi (~> 1.15)
+      mini_portile2 (~> 2.6)
+      rake (> 12)
+    rspec (3.12.0)
+      rspec-core (~> 3.12.0)
+      rspec-expectations (~> 3.12.0)
+      rspec-mocks (~> 3.12.0)
+    rspec-core (3.12.0)
+      rspec-support (~> 3.12.0)
+    rspec-expectations (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-mocks (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-mocks (3.12.0)
       diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.11.0)
-    rspec-support (3.11.1)
+      rspec-support (~> 3.12.0)
+    rspec-support (3.12.0)
     simplecov (0.21.2)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
@@ -55,4 +63,4 @@ DEPENDENCIES
   simplecov
 
 BUNDLED WITH
-   2.3.22
+   2.3.26

data/karafka-core.gemspec

@@ -15,7 +15,9 @@ Gem::Specification.new do |spec|
   spec.summary     = 'Karafka ecosystem core modules'
   spec.description = 'A toolset of small support modules used throughout the Karafka ecosystem'
   spec.licenses    = %w[MIT]
+
   spec.add_dependency 'concurrent-ruby', '>= 1.1'
+  spec.add_dependency 'rdkafka', '>= 0.12'
 
   spec.required_ruby_version = '>= 2.6.0'
 
@@ -28,7 +30,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = %w[lib]
 
   spec.metadata = {
+    'funding_uri' => 'https://karafka.io/#become-pro',
+    'homepage_uri' => 'https://karafka.io',
+    'changelog_uri' => 'https://github.com/karafka/karafka-core/blob/master/CHANGELOG.md',
+    'bug_tracker_uri' => 'https://github.com/karafka/karafka-core/issues',
     'source_code_uri' => 'https://github.com/karafka/karafka-core',
+    'documentation_uri' => 'https://karafka.io/docs',
     'rubygems_mfa_required' => 'true'
   }
 end

data/lib/karafka/core/instrumentation/callbacks_manager.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Karafka instrumentation related shared components
+    module Instrumentation
+      # This manager allows us to register multiple callbacks into a hook that is suppose to support
+      # a single callback
+      class CallbacksManager
+        # @return [::Karafka::Core::Instrumentation::CallbacksManager]
+        def initialize
+          @callbacks = Concurrent::Hash.new
+        end
+
+        # Invokes all the callbacks registered one after another
+        #
+        # @param args [Object] any args that should go to the callbacks
+        # @note We do not use `#each_value` here on purpose. With it being used, we cannot dispatch
+        #   callbacks and add new at the same time. Since we don't know when and in what thread
+        #   things are going to be added to the manager, we need to extract values into an array and
+        #   run it. That way we can add new things the same time.
+        def call(*args)
+          @callbacks.values.each { |callback| callback.call(*args) }
+        end
+
+        # Adds a callback to the manager
+        #
+        # @param id [String] id of the callback (used when deleting it)
+        # @param callable [#call] object that responds to a `#call` method
+        def add(id, callable)
+          @callbacks[id] = callable
+        end
+
+        # Removes the callback from the manager
+        # @param id [String] id of the callback we want to remove
+        def delete(id)
+          @callbacks.delete(id)
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/instrumentation.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # All the instrumentation shared across Karafka ecosystem
+    module Instrumentation
+      class << self
+        # Builds a manager for statistics callbacks
+        # @return [WaterDrop::CallbacksManager]
+        def statistics_callbacks
+          @statistics_callbacks ||= CallbacksManager.new
+        end
+
+        # Builds a manager for error callbacks
+        # @return [WaterDrop::CallbacksManager]
+        def error_callbacks
+          @error_callbacks ||= CallbacksManager.new
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/monitoring/notifications.rb

@@ -20,14 +20,17 @@ module Karafka
         private_constant :EMPTY_HASH
 
         def initialize
-          @listeners = Concurrent::Map.new { |k, v| k[v] = Concurrent::Array.new }
+          @listeners = Concurrent::Map.new do |k, v|
+            k.compute_if_absent(v) { Concurrent::Array.new }
+          end
+
           # This allows us to optimize the method calling lookups
           @events_methods_map = Concurrent::Map.new
         end
 
         # Registers a new event on which we can publish
         #
-        # @param event_id [String, Symbol] event id
+        # @param event_id [String] event id
         def register_event(event_id)
           @listeners[event_id]
           @events_methods_map[event_id] = :"on_#{event_id.to_s.tr('.', '_')}"
@@ -73,7 +76,7 @@ module Karafka
         # Allows for code instrumentation
         # Runs the provided code and sends the instrumentation details to all registered listeners
         #
-        # @param event_id [String, Symbol] id of the event
+        # @param event_id [String] id of the event
         # @param payload [Hash] payload for the instrumentation
         # @param block [Proc] instrumented code
         # @return [Object] whatever the provided block (if any) returns
@@ -83,6 +86,9 @@ module Karafka
         #     sleep(1)
         #   end
         def instrument(event_id, payload = EMPTY_HASH, &block)
+          # Allow for instrumentation of only events we registered
+          raise EventNotRegistered, event_id unless @listeners.key?(event_id)
+
           result, time = measure_time_taken(&block) if block_given?
 
           event = Event.new(

data/lib/karafka/core/patches/rdkafka/bindings.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Karafka
+  module Core
+    # Patches to dependencies and components
+    module Patches
+      # Patches to rdkafka
+      module Rdkafka
+        # Extends `Rdkafka::Bindings` with some extra methods and updates callbacks that we intend
+        # to work with in a bit different way than rdkafka itself
+        module Bindings
+          class << self
+            # Add extra methods that we need
+            # @param mod [::Rdkafka::Bindings] rdkafka bindings module
+            def included(mod)
+              mod.attach_function :rd_kafka_name, [:pointer], :string
+
+              # Default rdkafka setup for errors doest not propagate client details, thus it always
+              # publishes all the stuff for all rdkafka instances. We change that by providing
+              # function that fetches the instance name, allowing us to have better notifications
+              mod.send(:remove_const, :ErrorCallback)
+              mod.const_set(:ErrorCallback, build_error_callback)
+            end
+
+            # @return [FFI::Function] overwritten callback function
+            def build_error_callback
+              FFI::Function.new(
+                :void, %i[pointer int string pointer]
+              ) do |client_prr, err_code, reason, _opaque|
+                return nil unless ::Rdkafka::Config.error_callback
+
+                name = ::Rdkafka::Bindings.rd_kafka_name(client_prr)
+
+                error = ::Rdkafka::RdkafkaError.new(err_code, broker_message: reason)
+
+                ::Rdkafka::Config.error_callback.call(name, error)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/karafka/core/version.rb

@@ -4,6 +4,6 @@ module Karafka
   module Core
     # Current Karafka::Core version
     # We follow the versioning schema of given Karafka version
-    VERSION = '2.0.3'
+    VERSION = '2.0.5'
   end
 end

data/lib/karafka-core.rb

@@ -2,24 +2,44 @@
 
 %w[
   yaml
+  rdkafka
+
   concurrent/map
   concurrent/hash
   concurrent/array
+
   karafka/core
   karafka/core/version
+
   karafka/core/monitoring
   karafka/core/monitoring/event
   karafka/core/monitoring/monitor
   karafka/core/monitoring/notifications
   karafka/core/monitoring/statistics_decorator
+
   karafka/core/configurable
   karafka/core/configurable/leaf
   karafka/core/configurable/node
+
   karafka/core/contractable/contract
   karafka/core/contractable/result
   karafka/core/contractable/rule
+
+  karafka/core/instrumentation
+  karafka/core/instrumentation/callbacks_manager
+
+  karafka/core/patches/rdkafka/bindings
 ].each { |dependency| require dependency }
 
 # Karafka framework main namespace
 module Karafka
 end
+
+# Patch rdkafka
+::Rdkafka::Bindings.include(::Karafka::Core::Patches::Rdkafka::Bindings)
+
+# Rdkafka uses a single global callback for things. We bypass that by injecting a manager for
+# each callback type. Callback manager allows us to register more than one callback
+# @note Those managers are also used by Karafka for consumer related statistics
+::Rdkafka::Config.statistics_callback = ::Karafka::Core::Instrumentation.statistics_callbacks
+::Rdkafka::Config.error_callback = ::Karafka::Core::Instrumentation.error_callbacks

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: karafka-core
 version: !ruby/object:Gem::Version
-  version: 2.0.3
+  version: 2.0.5
 platform: ruby
 authors:
 - Maciej Mensfeld
@@ -35,7 +35,7 @@ cert_chain:
   Qf04B9ceLUaC4fPVEz10FyobjaFoY4i32xRto3XnrzeAgfEe4swLq8bQsR3w/EF3
   MGU0FeSV2Yj7Xc2x/7BzLK8xQn5l7Yy75iPF+KP3vVmDHnNl
   -----END CERTIFICATE-----
-date: 2022-10-13 00:00:00.000000000 Z
+date: 2022-12-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -51,6 +51,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '1.1'
+- !ruby/object:Gem::Dependency
+  name: rdkafka
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.12'
 description: A toolset of small support modules used throughout the Karafka ecosystem
 email:
 - contact@karafka.io
@@ -61,6 +75,7 @@ files:
 - ".coditsu/ci.yml"
 - ".console_irbrc"
 - ".diffend.yml"
+- ".github/FUNDING.yml"
 - ".github/ISSUE_TEMPLATE/bug_report.md"
 - ".github/ISSUE_TEMPLATE/feature_request.md"
 - ".github/workflows/ci.yml"
@@ -86,18 +101,26 @@ files:
 - lib/karafka/core/contractable/contract.rb
 - lib/karafka/core/contractable/result.rb
 - lib/karafka/core/contractable/rule.rb
+- lib/karafka/core/instrumentation.rb
+- lib/karafka/core/instrumentation/callbacks_manager.rb
 - lib/karafka/core/monitoring.rb
 - lib/karafka/core/monitoring/event.rb
 - lib/karafka/core/monitoring/monitor.rb
 - lib/karafka/core/monitoring/notifications.rb
 - lib/karafka/core/monitoring/statistics_decorator.rb
+- lib/karafka/core/patches/rdkafka/bindings.rb
 - lib/karafka/core/version.rb
 - log/.gitkeep
 homepage: https://karafka.io
 licenses:
 - MIT
 metadata:
+  funding_uri: https://karafka.io/#become-pro
+  homepage_uri: https://karafka.io
+  changelog_uri: https://github.com/karafka/karafka-core/blob/master/CHANGELOG.md
+  bug_tracker_uri: https://github.com/karafka/karafka-core/issues
   source_code_uri: https://github.com/karafka/karafka-core
+  documentation_uri: https://karafka.io/docs
   rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
@@ -114,7 +137,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.26
 signing_key: 
 specification_version: 4
 summary: Karafka ecosystem core modules